00010000:02000000:1.1F:1783518104.418461:0:35274:0:(ldlm_lockd.c:390:__ldlm_add_waiting_lock()) *** cfs_fail_loc=512, val=0*** 00010000:02000000:0.1F:1783518105.461647:0:6482:0:(ldlm_lockd.c:390:__ldlm_add_waiting_lock()) *** cfs_fail_loc=512, val=0*** 00010000:02000000:0.1:1783518106.485520:0:6482:0:(ldlm_lockd.c:390:__ldlm_add_waiting_lock()) *** cfs_fail_loc=512, val=0*** 00010000:02000000:0.1:1783518107.509535:0:6482:0:(ldlm_lockd.c:390:__ldlm_add_waiting_lock()) *** cfs_fail_loc=512, val=0*** 00010000:02000000:0.1:1783518125.501405:0:35284:0:(ldlm_lockd.c:390:__ldlm_add_waiting_lock()) *** cfs_fail_loc=512, val=0*** 00010000:02000000:0.1:1783518126.517612:0:6482:0:(ldlm_lockd.c:390:__ldlm_add_waiting_lock()) *** cfs_fail_loc=512, val=0*** 00010000:02000000:0.1:1783518127.541644:0:6482:0:(ldlm_lockd.c:390:__ldlm_add_waiting_lock()) *** cfs_fail_loc=512, val=0*** 00010000:02000000:0.1:1783518128.565518:0:6482:0:(ldlm_lockd.c:390:__ldlm_add_waiting_lock()) *** cfs_fail_loc=512, val=0*** 00010000:02000000:0.1:1783518129.568338:0:35284:0:(ldlm_lockd.c:390:__ldlm_add_waiting_lock()) *** cfs_fail_loc=512, val=0*** 00010000:02000000:2.1F:1783518130.614888:0:6482:0:(ldlm_lockd.c:390:__ldlm_add_waiting_lock()) *** cfs_fail_loc=512, val=0*** 00010000:02000000:2.1:1783518131.637627:0:6482:0:(ldlm_lockd.c:390:__ldlm_add_waiting_lock()) *** cfs_fail_loc=512, val=0*** 00010000:02000000:2.1:1783518132.661628:0:6482:0:(ldlm_lockd.c:390:__ldlm_add_waiting_lock()) *** cfs_fail_loc=512, val=0*** 00010000:02000000:1.1:1783518138.809809:0:20140:0:(ldlm_lockd.c:391:__ldlm_add_waiting_lock()) *** cfs_fail_loc=511, val=0*** 00010000:02000000:0.1:1783518138.817060:0:20140:0:(ldlm_lockd.c:391:__ldlm_add_waiting_lock()) *** cfs_fail_loc=511, val=0*** 00010000:02000000:3.1F:1783518151.611668:0:20415:0:(ldlm_lockd.c:391:__ldlm_add_waiting_lock()) *** cfs_fail_loc=511, val=0*** 00010000:00010000:3.0:1783520930.924042:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7a9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.924063:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc0242b4 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x7a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.924120:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000ed94e4ca/0xd5401cbabc02158f lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b4b expref: 374 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.924155:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000ed94e4ca/0xd5401cbabc02158f lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b4b expref: 374 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.924195:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515302a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0000_UUID:178496:1870152992955392:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520930.924219:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.924227:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000e3859524/0xd5401cbabc024298 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.924248:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.924257:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000e3859524/0xd5401cbabc024298 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.924290:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000e3859524/0xd5401cbabc024298 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.924315:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000e3859524/0xd5401cbabc024298 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.924332:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000e3859524/0xd5401cbabc024298 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.924601:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.924605:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.924608:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.924718:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152992955776 00000100:00100000:3.0:1783520930.924723:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152992955776 00000100:00100000:3.0:1783520930.924757:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43705 00000100:00100000:3.0:1783520930.924768:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+659:3625:x1870152992955776:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520930.924781:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:1964 count 1 00010000:00010000:3.0:1783520930.924793:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc0242c9 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7ac:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.924805:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 0000000055b06cce/0xd5401cbabc0215a4 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x7ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b75 expref: 373 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.924812:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7ac:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520930.924816:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7ac:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.924820:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc0242c9 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x7ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.924841:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 0000000055b06cce/0xd5401cbabc0215a4 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b75 expref: 373 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.924851:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 0000000055b06cce/0xd5401cbabc0215a4 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b75 expref: 373 pid: 12734 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.924867:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515302300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0000_UUID:35282:1870152992956032:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520930.924900:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350b830900 going to sleep for 11 seconds 00000100:00100000:3.0:1783520930.924921:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505109840 going to sleep for 11 seconds 00000100:00100000:3.0:1783520930.924932:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002299008 00000100:00100000:3.0:1783520930.924936:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002299008 00010000:00010000:3.0:1783520930.924997:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 0000000091e7ae34/0xd5401cbabc02156c lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b21 expref: 374 pid: 35275 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.925018:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35297 00000100:00100000:3.0:1783520930.925026:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435151a5c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+374:2388:x1870153002299008:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.925035:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.925041:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 0000000091e7ae34/0xd5401cbabc02156c lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7a6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b21 expref: 374 pid: 35275 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.925056:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 0000000091e7ae34/0xd5401cbabc02156c lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7a6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b21 expref: 374 pid: 35275 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.925075:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000091e7ae34/0xd5401cbabc02156c lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7a6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b21 expref: 374 pid: 35275 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.925172:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7a6:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.925184:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7a6:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.925202:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000c8b052a2/0xd5401cbabc02429f lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.925254:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000c8b052a2/0xd5401cbabc02429f lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.925328:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.925390:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435151a5c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+373:2388:x1870153002299008:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 362us (654us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.925406:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35297 00010000:00010000:3.0:1783520930.926476:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc0242a6 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.926631:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc0242a6 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.926640:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc0242a6 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.926647:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc0242a6 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.926665:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520930.926805:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+662:3627:x1870152992954496:12345-0@lo:6:osp-syn-0-0.0 Request processed in 3553us (3720us total) trans 4294977183 rc 0/0 00000100:00100000:3.0:1783520930.926812:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43700 00000100:00100000:3.0:1783520930.927226:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515302300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0000_UUID:35282:1870152992956032:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520930.927252:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc0242c9 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.927698:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002299520 00000100:00100000:3.0:1783520930.927704:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002299520 00010000:00010000:3.0:1783520930.927726:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 0000000054cfa313/0xd5401cbabc021596 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x7aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b59 expref: 375 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.927764:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002299776 00000100:00100000:3.0:1783520930.927769:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002299776 00010000:00010000:3.0:1783520930.927786:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000000e923552/0xd5401cbabc02159d lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x7ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b67 expref: 376 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.927900:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002300032 00000100:00100000:3.0:1783520930.927908:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002300032 00010000:00010000:3.0:1783520930.927942:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 0000000084f10b8c/0xd5401cbabc0215b2 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x7ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b91 expref: 377 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.928027:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35301 00000100:00100000:3.0:1783520930.928104:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357d850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+376:2387:x1870153002299520:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.928129:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.928140:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 0000000054cfa313/0xd5401cbabc021596 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7aa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b59 expref: 376 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.928184:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 0000000054cfa313/0xd5401cbabc021596 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7aa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b59 expref: 376 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.928228:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000054cfa313/0xd5401cbabc021596 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7aa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b59 expref: 375 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.928284:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7aa:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.928298:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7aa:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.928314:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000007aa431df/0xd5401cbabc0242bb lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.928381:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000007aa431df/0xd5401cbabc0242bb lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.928475:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.928523:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357d850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+371:2387:x1870153002299520:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 474us (895us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.928549:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35301 00010000:00010000:3.0:1783520930.928627:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.928642:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc0242ad lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.928701:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.928719:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc0242ad lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.928760:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc0242ad lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.928804:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc0242ad lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.928881:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc0242ad lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.928950:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.928953:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.928957:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.928969:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc0242c9 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.928978:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc0242c9 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.928989:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc0242c9 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.928998:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc0242c9 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.929053:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520930.929153:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+663:3625:x1870152992955776:12345-0@lo:6:osp-syn-0-0.0 Request processed in 4384us (4605us total) trans 4294977185 rc 0/0 00000100:00100000:3.0:1783520930.929164:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43705 00000100:00100000:3.0:1783520930.929191:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152992956928 00000100:00100000:3.0:1783520930.929198:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152992956928 00000100:00100000:3.0:1783520930.929244:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43709 00000100:00100000:3.0:1783520930.929256:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+664:3625:x1870152992956928:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520930.929278:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:1968 count 1 00010000:00010000:3.0:1783520930.929299:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc0242e5 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7b0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.929325:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 000000006c5d691f/0xd5401cbabc0215c7 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x7b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bad expref: 365 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.929343:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7b0:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520930.929349:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7b0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.929355:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc0242e5 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x7b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.929418:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 000000006c5d691f/0xd5401cbabc0215c7 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bad expref: 365 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.929430:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 000000006c5d691f/0xd5401cbabc0215c7 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bad expref: 365 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.929453:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515302300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0000_UUID:35282:1870152992957056:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520930.929521:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57b40 going to sleep for 11 seconds 00000100:00100000:3.0:1783520930.929618:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002300160 00000100:00100000:3.0:1783520930.929627:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002300160 00010000:00010000:3.0:1783520930.929648:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000a5557257/0xd5401cbabc0215c0 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x7af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b9f expref: 366 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.929676:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35306 00000100:00100000:3.0:1783520930.929690:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c5c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+366:2388:x1870153002300160:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.929702:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.929710:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000a5557257/0xd5401cbabc0215c0 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x7af:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b9f expref: 366 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.929737:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000a5557257/0xd5401cbabc0215c0 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7af:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b9f expref: 366 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.929764:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000a5557257/0xd5401cbabc0215c0 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7af:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b9f expref: 366 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.929772:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7af:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.929777:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7af:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.929787:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000be742370/0xd5401cbabc0242de lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7af:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.929818:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000be742370/0xd5401cbabc0242de lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.929849:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.929885:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c5c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+365:2388:x1870153002300160:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 198us (401us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.929913:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35306 00010000:00010000:3.0:1783520930.929962:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.929968:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000be742370/0xd5401cbabc0242de lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.929989:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.929996:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000be742370/0xd5401cbabc0242de lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.930040:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000be742370/0xd5401cbabc0242de lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.930091:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000be742370/0xd5401cbabc0242de lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.930135:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000be742370/0xd5401cbabc0242de lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.930218:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.930225:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000005120a6/0xd5401cbabc0242c2 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.930244:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.930248:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000005120a6/0xd5401cbabc0242c2 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.930255:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000005120a6/0xd5401cbabc0242c2 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.930264:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000005120a6/0xd5401cbabc0242c2 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.930271:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000005120a6/0xd5401cbabc0242c2 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.930308:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.930313:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.930318:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.930435:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.930464:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516682680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+364:2387:x1870153002300032:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 1831us (2615us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.930477:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35305 00000100:00100000:3.0:1783520930.930582:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152992957696 00000100:00100000:3.0:1783520930.930588:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152992957696 00000100:00100000:3.0:1783520930.930642:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43713 00000100:00100000:3.0:1783520930.930660:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c56bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+667:3625:x1870152992957696:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520930.930676:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:1972 count 1 00010000:00010000:3.0:1783520930.930692:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024301 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7b4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.930705:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 000000004d083dde/0xd5401cbabc0215e3 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x7b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1be5 expref: 363 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.930713:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7b4:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520930.930716:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7b4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.930724:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024301 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x7b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.930761:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 000000004d083dde/0xd5401cbabc0215e3 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1be5 expref: 363 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.930782:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 000000004d083dde/0xd5401cbabc0215e3 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1be5 expref: 363 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.930796:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515303480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0000_UUID:8410:1870152992958080:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520930.930842:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350b830f00 going to sleep for 11 seconds 00000100:00100000:3.0:1783520930.933565:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515302300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0000_UUID:35282:1870152992957056:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520930.933609:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc0242e5 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.933656:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002300544 00000100:00100000:3.0:1783520930.933663:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002300544 00010000:00010000:3.0:1783520930.933687:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000594bbfb6/0xd5401cbabc0215f1 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c01 expref: 366 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.933755:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002300672 00000100:00100000:3.0:1783520930.933761:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002300672 00010000:00010000:3.0:1783520930.933782:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000006c5d691f/0xd5401cbabc0215c7 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bad expref: 367 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.933839:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35308 00000100:00100000:3.0:1783520930.933858:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357ec50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+366:2387:x1870153002300416:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.933877:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.933891:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000c137f516/0xd5401cbabc0215ce lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7b2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bc9 expref: 366 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.933905:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000c137f516/0xd5401cbabc0215ce lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7b2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bc9 expref: 366 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.933924:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c137f516/0xd5401cbabc0215ce lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7b2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bc9 expref: 366 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.933953:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7b2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.933963:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7b2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.933971:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000f07fd018/0xd5401cbabc0242f3 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.933984:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000f07fd018/0xd5401cbabc0242f3 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.934022:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.934067:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357ec50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+367:2387:x1870153002300416:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 213us (506us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.934089:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35308 00000100:00100000:3.0:1783520930.934174:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35312 00010000:00010000:3.0:1783520930.934396:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc0242e5 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.934470:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc0242e5 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.934507:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc0242e5 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.934556:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc0242e5 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.934604:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520930.934617:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.934623:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc0242e5 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.934657:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784d000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.934668:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc0242e5 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.934717:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc0242e5 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.934736:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc0242e5 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.934760:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc0242e5 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.934869:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.934874:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.934880:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.934986:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+672:3625:x1870152992956928:12345-0@lo:6:osp-syn-0-0.0 Request processed in 5731us (5845us total) trans 4294977194 rc 0/0 00000100:00100000:3.0:1783520930.934995:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43709 00000100:00100000:3.0:1783520930.935043:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515303480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0000_UUID:8410:1870152992958080:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520930.935067:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024301 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.935131:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c56b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+360:2388:x1870153002300800:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.935144:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.935154:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000002caaa25f/0xd5401cbabc0215d5 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7b1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bbb expref: 360 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.935179:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000002caaa25f/0xd5401cbabc0215d5 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7b1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bbb expref: 360 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.935208:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000002caaa25f/0xd5401cbabc0215d5 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7b1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bbb expref: 360 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.935230:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7b1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.935234:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7b1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.935242:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc0242ec lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.935258:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc0242ec lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.935309:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.935340:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c56b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+358:2388:x1870153002300800:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 949us (1471us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.935439:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35312 00000100:00100000:3.0:1783520930.935485:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43718 00000100:00100000:3.0:1783520930.935504:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363566fc50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+672:3625:x1870152992959104:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520930.935526:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:1977 count 1 00010000:00010000:3.0:1783520930.935543:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024324 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7b9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.935566:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000016cc81f/0xd5401cbabc021606 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x7b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c2b expref: 357 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.935580:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7b9:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520930.935600:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7b9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.935609:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024324 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x7b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.935634:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000016cc81f/0xd5401cbabc021606 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c2b expref: 357 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.935659:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000016cc81f/0xd5401cbabc021606 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c2b expref: 357 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.935679:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515303480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0000_UUID:8409:1870152992959488:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520930.935732:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57b40 going to sleep for 11 seconds 00000100:00100000:3.0:1783520930.935779:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c568e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+357:2388:x1870153002301056:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.935792:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.935799:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000004d083dde/0xd5401cbabc0215e3 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7b4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1be5 expref: 357 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.935820:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000004d083dde/0xd5401cbabc0215e3 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7b4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1be5 expref: 357 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.935848:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000004d083dde/0xd5401cbabc0215e3 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7b4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1be5 expref: 357 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.935867:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7b4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.935871:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7b4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.935874:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024301 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.935880:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024301 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.935905:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520930.935922:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024301 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.935930:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024301 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.935936:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024301 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.935941:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024301 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.935956:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520930.935964:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.935969:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024301 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.935984:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784ca00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.935992:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024301 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.936009:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024301 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.936039:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024301 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.936140:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024301 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.936198:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.936202:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.936220:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.936231:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000f07fd018/0xd5401cbabc0242f3 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.936238:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.936243:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000f07fd018/0xd5401cbabc0242f3 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.936266:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000f07fd018/0xd5401cbabc0242f3 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.936280:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000f07fd018/0xd5401cbabc0242f3 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.936297:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000f07fd018/0xd5401cbabc0242f3 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.936315:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.936319:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.936323:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.936481:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c56bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+676:3625:x1870152992957696:12345-0@lo:6:osp-syn-0-0.0 Request processed in 5821us (5923us total) trans 4294977198 rc 0/0 00000100:00100000:3.0:1783520930.936504:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43713 00000100:00100000:3.0:1783520930.936538:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43720 00000100:00100000:3.0:1783520930.936553:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c56e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+675:3627:x1870152992959616:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520930.936573:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:1979 count 1 00010000:00010000:3.0:1783520930.936587:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024332 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7bb:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.936627:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 0000000006195c45/0xd5401cbabc021614 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x7bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c47 expref: 356 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.936639:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7bb:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520930.936645:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7bb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.936654:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024332 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x7bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.936682:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 0000000006195c45/0xd5401cbabc021614 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c47 expref: 356 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.936699:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 0000000006195c45/0xd5401cbabc021614 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c47 expref: 356 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.936769:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515302300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0000_UUID:8410:1870152992960128:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520930.936824:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350b830f00 going to sleep for 11 seconds 00000100:00100000:3.0:1783520930.936864:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c568e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+356:2388:x1870153002301056:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 2403us (2945us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.936873:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35313 00000100:00100000:3.0:1783520930.936921:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152992960384 00000100:00100000:3.0:1783520930.936932:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152992960384 00000100:00100000:3.0:1783520930.936975:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43723 00000100:00100000:3.0:1783520930.936989:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c56fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+677:3627:x1870152992960384:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520930.937007:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:1982 count 1 00010000:00010000:3.0:1783520930.937152:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc024347 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7be:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.937180:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000e77ba0e7/0xd5401cbabc021630 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x7be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c71 expref: 355 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.937195:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7be:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520930.937203:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7be:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.937210:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc024347 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x7be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.937244:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000e77ba0e7/0xd5401cbabc021630 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c71 expref: 355 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.937262:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000e77ba0e7/0xd5401cbabc021630 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c71 expref: 355 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.937284:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515300e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0000_UUID:20415:1870152992960896:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520930.937423:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af10c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520930.937767:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002301440 00000100:00100000:3.0:1783520930.937775:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002301440 00010000:00010000:3.0:1783520930.937810:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000016cc81f/0xd5401cbabc021606 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c2b expref: 358 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.937918:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35317 00000100:00100000:3.0:1783520930.937938:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c56f100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+358:2387:x1870153002301440:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.937960:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.937977:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000016cc81f/0xd5401cbabc021606 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7b9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c2b expref: 358 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.938000:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000016cc81f/0xd5401cbabc021606 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7b9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c2b expref: 358 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.938062:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000016cc81f/0xd5401cbabc021606 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7b9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c2b expref: 358 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.938102:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7b9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.938126:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7b9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.938159:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024324 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.938220:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024324 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.938288:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.938328:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c56f100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+355:2387:x1870153002301440:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 397us (627us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.938412:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35317 00010000:00010000:3.0:1783520930.938485:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024324 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.938497:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024324 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.938510:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024324 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.938522:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024324 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.938555:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520930.938567:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.938575:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024324 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.938614:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784ce00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.938620:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024324 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.938839:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024324 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.938856:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024324 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.938876:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024324 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.938916:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.938920:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.938923:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.939059:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363566fc50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+679:3625:x1870152992959104:12345-0@lo:6:osp-syn-0-0.0 Request processed in 3558us (3881us total) trans 4294977202 rc 0/0 00000100:00100000:3.0:1783520930.939068:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43718 00000100:00100000:3.0:1783520930.939427:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152992961024 00000100:00100000:3.0:1783520930.939434:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152992961024 00000100:00100000:3.0:1783520930.939473:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152992961408 00000100:00100000:3.0:1783520930.939479:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152992961408 00000100:00100000:3.0:1783520930.939604:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002301696 00000100:00100000:3.0:1783520930.939611:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002301696 00010000:00010000:3.0:1783520930.939638:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000b8e222d6/0xd5401cbabc02161b lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c55 expref: 353 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.939846:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002301952 00000100:00100000:3.0:1783520930.939854:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002301952 00010000:00010000:3.0:1783520930.939949:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000fec49f30/0xd5401cbabc021622 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c63 expref: 354 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.940020:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002301824 00000100:00100000:3.0:1783520930.940027:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002301824 00010000:00010000:3.0:1783520930.940056:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 0000000006195c45/0xd5401cbabc021614 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7bb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c47 expref: 356 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.940195:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35319 00000100:00100000:3.0:1783520930.940207:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c568a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+356:2387:x1870153002301952:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.940227:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.940240:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000fec49f30/0xd5401cbabc021622 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7bd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c63 expref: 356 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.940277:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000fec49f30/0xd5401cbabc021622 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7bd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c63 expref: 356 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.940435:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000fec49f30/0xd5401cbabc021622 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7bd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c63 expref: 354 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.940456:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7bd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.940461:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7bd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.940468:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc024340 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.940478:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc024340 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.940607:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.940690:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c568a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+353:2387:x1870153002301952:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 482us (1397us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.940728:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35319 00000100:00100000:3.0:1783520930.940953:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35322 00000100:00100000:3.0:1783520930.940971:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357cc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+351:2388:x1870153002302080:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.940987:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.940997:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000bce16992/0xd5401cbabc021637 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7bf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c7f expref: 350 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.941155:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000bce16992/0xd5401cbabc021637 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7bf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c7f expref: 349 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.941307:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000bce16992/0xd5401cbabc021637 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7bf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c7f expref: 348 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.941354:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7bf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.941437:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7bf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.941443:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000005cc63f31/0xd5401cbabc02434e lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.941460:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000005cc63f31/0xd5401cbabc02434e lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.941532:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.941574:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357cc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+349:2388:x1870153002302080:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 609us (1870us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.941603:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35322 00000100:00100000:3.0:1783520930.941669:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35325 00000100:00100000:3.0:1783520930.941688:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c56f480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+348:2387:x1870153002302592:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.941704:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.941713:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 0000000035373bb0/0xd5401cbabc02164c lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7c2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ca9 expref: 348 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.941741:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 0000000035373bb0/0xd5401cbabc02164c lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7c2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ca9 expref: 348 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.941785:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000035373bb0/0xd5401cbabc02164c lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7c2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ca9 expref: 348 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.941814:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7c2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.941825:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7c2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.941830:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc024363 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.941852:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc024363 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.941899:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.941933:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c56f480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+346:2387:x1870153002302592:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 245us (463us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.941963:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35325 00000100:00100000:3.0:1783520930.942146:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c56ea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+345:2387:x1870153002302464:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 453us (795us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.942171:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35324 00010000:00010000:3.0:1783520930.942411:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc024347 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.942458:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc024347 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.942510:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc024347 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.942532:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc024347 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.942765:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.942943:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to regular blp list, 0 blwis in pool 00000100:00100000:3.0:1783520930.943053:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c56fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+686:3627:x1870152992960384:12345-0@lo:6:osp-syn-0-0.0 Request processed in 6062us (6208us total) trans 4294977207 rc 0/0 00000100:00100000:3.0:1783520930.943071:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43723 00010000:00010000:3.0:1783520930.943151:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000005f11335d/0xd5401cbabc024355 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.943211:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000005f11335d/0xd5401cbabc024355 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.943228:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000005f11335d/0xd5401cbabc024355 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.943237:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000005f11335d/0xd5401cbabc024355 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.943331:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520930.943492:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516312680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+686:3626:x1870152992961152:12345-0@lo:6:osp-syn-0-0.0 Request processed in 4310us (4417us total) trans 4294977209 rc 0/0 00000100:00100000:3.0:1783520930.943507:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43725 00000100:00100000:3.0:1783520930.943545:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43732 00000100:00100000:3.0:1783520930.943556:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c56a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+686:3626:x1870152992962816:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520930.943574:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:1992 count 1 00010000:00010000:3.0:1783520930.943604:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024386 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7c8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.943746:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000790f7f21/0xd5401cbabc021676 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x7c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cfd expref: 344 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.943780:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7c8:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520930.943850:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7c8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.943855:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024386 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x7c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.943886:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000790f7f21/0xd5401cbabc021676 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cfd expref: 344 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.943897:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000790f7f21/0xd5401cbabc021676 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cfd expref: 344 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.943921:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515302300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0000_UUID:35274:1870152992963200:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520930.943961:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af1000 going to sleep for 11 seconds 00000100:00100000:3.0:1783520930.944051:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152992962688 00000100:00100000:3.0:1783520930.944063:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152992962688 00000100:00100000:3.0:1783520930.944105:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43734 00000100:00100000:3.0:1783520930.944119:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436342bad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+688:3628:x1870152992962688:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520930.944157:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:1991 count 1 00010000:00010000:3.0:1783520930.944268:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024394 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7c7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.944300:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 000000004c162c03/0xd5401cbabc02166f lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x7c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cef expref: 345 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.944308:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7c7:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520930.944312:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7c7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.944322:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024394 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x7c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.944353:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 000000004c162c03/0xd5401cbabc02166f lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cef expref: 346 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.944394:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 000000004c162c03/0xd5401cbabc02166f lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cef expref: 346 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.944414:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515300e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0000_UUID:9151:1870152992963584:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520930.944465:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af19c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520930.944502:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35327 00000100:00100000:3.0:1783520930.944519:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c5880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+346:2388:x1870153002302848:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.944543:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.944557:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000cc907ce9/0xd5401cbabc02165a lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x7c4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cc5 expref: 346 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.944619:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000cc907ce9/0xd5401cbabc02165a lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7c4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cc5 expref: 346 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.944642:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7c4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.944649:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7c4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.944665:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000c42e258a/0xd5401cbabc024371 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7c4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.944698:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000c42e258a/0xd5401cbabc024371 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7c4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.944722:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.944828:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c5880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+345:2388:x1870153002302848:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 239us (603us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.944862:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35327 00010000:00010000:3.0:1783520930.944931:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.944941:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000a5d93cec/0xd5401cbabc02436a lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.944960:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.944966:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000a5d93cec/0xd5401cbabc02436a lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.944983:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000a5d93cec/0xd5401cbabc02436a lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.945069:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000a5d93cec/0xd5401cbabc02436a lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.945098:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000a5d93cec/0xd5401cbabc02436a lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.945137:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.945142:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.945146:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.945326:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.945422:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.945489:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515302300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0000_UUID:35274:1870152992963200:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520930.945529:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024386 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.945690:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35330 00000100:00100000:3.0:1783520930.945711:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515303480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+342:2388:x1870153002303104:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.945725:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.945776:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000afd95d60/0xd5401cbabc021668 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7c6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ce1 expref: 342 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.945810:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000afd95d60/0xd5401cbabc021668 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7c6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ce1 expref: 341 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.945845:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000afd95d60/0xd5401cbabc021668 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7c6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ce1 expref: 341 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.945871:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7c6:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.945878:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7c6:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.945883:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 0000000000a37bb7/0xd5401cbabc02437f lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.945898:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000000a37bb7/0xd5401cbabc02437f lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.945932:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.946012:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515303480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+340:2388:x1870153002303104:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 306us (587us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.946028:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35330 00010000:00010000:3.0:1783520930.946057:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024386 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.946104:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024386 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.946167:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024386 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.946216:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024386 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.946274:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520930.946287:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.946295:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024386 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.946345:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784de00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.946382:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024386 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.946399:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024386 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.946447:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024386 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.946460:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024386 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.946503:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.946508:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.946518:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.947200:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c56a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+694:3626:x1870152992962816:12345-0@lo:6:osp-syn-0-0.0 Request processed in 3644us (3770us total) trans 4294977215 rc 0/0 00000100:00100000:3.0:1783520930.947232:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43732 00000100:00100000:3.0:1783520930.947260:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152992964864 00000100:00100000:3.0:1783520930.947265:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152992964864 00000100:00100000:3.0:1783520930.947307:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43740 00000100:00100000:3.0:1783520930.947320:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515304380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+694:3628:x1870152992964864:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520930.947417:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:1999 count 1 00010000:00010000:3.0:1783520930.947438:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0243be lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7cf:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.947484:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000bb7bab96/0xd5401cbabc0216a0 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x7cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d5f expref: 340 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.947512:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7cf:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520930.947522:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7cf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.947538:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0243be lrc: 4/0,1 mode: --/PW res: [0x280000401:0x7cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.947569:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000bb7bab96/0xd5401cbabc0216a0 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d5f expref: 340 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.947625:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000bb7bab96/0xd5401cbabc0216a0 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d5f expref: 339 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.947779:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515302300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0000_UUID:35274:1870152992964992:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520930.947835:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af1000 going to sleep for 11 seconds 00000100:00100000:3.0:1783520930.948145:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.948206:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.948715:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002304000 00000100:00100000:3.0:1783520930.948723:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002304000 00010000:00010000:3.0:1783520930.948748:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 0000000015150727/0xd5401cbabc0216a7 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d51 expref: 342 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.948774:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002303872 00000100:00100000:3.0:1783520930.948779:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002303872 00010000:00010000:3.0:1783520930.948856:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000004df0d43a/0xd5401cbabc02168b lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d27 expref: 343 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.948904:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35335 00000100:00100000:3.0:1783520930.948933:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515306680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+342:2390:x1870153002304000:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.948959:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.948978:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 0000000015150727/0xd5401cbabc0216a7 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7ce:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d51 expref: 342 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.949040:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 0000000015150727/0xd5401cbabc0216a7 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7ce:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d51 expref: 341 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.949150:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000015150727/0xd5401cbabc0216a7 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7ce:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d51 expref: 341 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.949191:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7ce:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.949200:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7ce:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.949207:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc0243b7 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.949225:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc0243b7 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.949330:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.949401:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515306680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+338:2390:x1870153002304000:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 435us (720us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.949415:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35335 00000100:00100000:3.0:1783520930.949485:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.949632:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.949720:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515302300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0000_UUID:35274:1870152992964992:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520930.950472:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0243be lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.950497:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.950555:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35338 00000100:00100000:3.0:1783520930.950568:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515300000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+336:2388:x1870153002304256:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.950580:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.950587:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000bb7bab96/0xd5401cbabc0216a0 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7cf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d5f expref: 336 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.950628:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000bb7bab96/0xd5401cbabc0216a0 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7cf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d5f expref: 336 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.950651:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000bb7bab96/0xd5401cbabc0216a0 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7cf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d5f expref: 336 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.950674:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7cf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.950695:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7cf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.950706:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0243be lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.950832:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0243be lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.950887:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.950928:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515300000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+335:2388:x1870153002304256:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 360us (1444us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.950950:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35338 00000100:00100000:3.0:1783520930.950981:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35339 00000100:00100000:3.0:1783520930.950998:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515300a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+334:2388:x1870153002304384:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.951027:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.951039:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000007db45e06/0xd5401cbabc0216ae lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7d0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d6d expref: 334 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.951061:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000007db45e06/0xd5401cbabc0216ae lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7d0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d6d expref: 334 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.951079:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000007db45e06/0xd5401cbabc0216ae lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7d0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d6d expref: 334 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.951097:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7d0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.951170:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7d0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.951180:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc0243c5 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.951189:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc0243c5 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.951225:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.951254:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515300a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+333:2388:x1870153002304384:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 260us (1626us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.951282:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35339 00010000:00010000:3.0:1783520930.951304:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0243be lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.951351:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0243be lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.951388:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0243be lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.951409:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0243be lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.951443:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:3.0:1783520930.951673:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515304380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+701:3628:x1870152992964864:12345-0@lo:6:osp-syn-0-0.0 Request processed in 4352us (4462us total) trans 4294977222 rc 0/0 00000100:00100000:3.0:1783520930.951686:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43740 00000100:00100000:3.0:1783520930.951713:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152992966272 00000100:00100000:3.0:1783520930.951724:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152992966272 00000100:00100000:3.0:1783520930.951821:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43746 00000100:00100000:3.0:1783520930.951836:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c568380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+700:3626:x1870152992966272:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520930.951855:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2005 count 1 00010000:00010000:3.0:1783520930.951873:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc0243e8 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7d5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.951904:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 000000001981ca1a/0xd5401cbabc0216d8 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x7d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1db3 expref: 331 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.951924:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7d5:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520930.951929:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7d5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.951933:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc0243e8 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x7d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.951968:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 000000001981ca1a/0xd5401cbabc0216d8 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1db3 expref: 331 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.951997:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 000000001981ca1a/0xd5401cbabc0216d8 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1db3 expref: 331 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.952024:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515302300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0000_UUID:35274:1870152992966784:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520930.952066:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af1600 going to sleep for 11 seconds 00000100:00100000:3.0:1783520930.952204:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.952310:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.952646:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35341 00000100:00100000:3.0:1783520930.952703:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515300700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+333:2387:x1870153002304512:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.952723:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.952731:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc0243a2 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.952752:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.952758:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc0243a2 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.952779:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc0243a2 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.952793:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc0243a2 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.952804:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc0243a2 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.952820:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.952824:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.952827:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.952839:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.952855:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000a8c23d0e/0xd5401cbabc0216c3 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7d2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d89 expref: 332 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.952896:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000a8c23d0e/0xd5401cbabc0216c3 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7d2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d89 expref: 331 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.952917:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000a8c23d0e/0xd5401cbabc0216c3 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7d2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d89 expref: 331 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.952932:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7d2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.952936:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7d2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.952941:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000001e04e98c/0xd5401cbabc0243cc lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.952951:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000001e04e98c/0xd5401cbabc0243cc lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.953016:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.953059:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515300700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+330:2387:x1870153002304512:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 390us (754us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.953080:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35341 00000100:00100000:3.0:1783520930.953329:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c5880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0000_UUID:35276:1870152992966144:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520930.953353:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc0243da lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.953410:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.953468:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515302d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+329:2388:x1870153002303616:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 4638us (5262us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.953481:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35333 00000100:00100000:3.0:1783520930.953795:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152992967296 00000100:00100000:3.0:1783520930.953802:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152992967296 00000100:00100000:3.0:1783520930.953845:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43750 00000100:00100000:3.0:1783520930.953855:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435130e3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+704:3625:x1870152992967296:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520930.953872:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2009 count 1 00010000:00010000:3.0:1783520930.953887:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024404 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7d9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.953899:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000915ce8fd/0xd5401cbabc0216f4 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x7d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1deb expref: 328 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.953908:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7d9:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520930.953912:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7d9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.953916:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024404 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x7d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.953938:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000915ce8fd/0xd5401cbabc0216f4 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1deb expref: 328 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.953952:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000915ce8fd/0xd5401cbabc0216f4 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1deb expref: 328 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.953961:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515303480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0000_UUID:35275:1870152992967680:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520930.953984:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af1000 going to sleep for 11 seconds 00000100:00100000:3.0:1783520930.954219:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152992967552 00000100:00100000:3.0:1783520930.954225:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152992967552 00000100:00100000:3.0:1783520930.954261:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43751 00000100:00100000:3.0:1783520930.954269:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cba8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+705:3627:x1870152992967552:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520930.954283:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2010 count 1 00010000:00010000:3.0:1783520930.954294:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc02440b lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7da:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.954443:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000a9bd3902/0xd5401cbabc0216fb lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x7da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1df9 expref: 328 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.954563:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7da:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520930.954568:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7da:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.954573:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc02440b lrc: 4/0,1 mode: --/PW res: [0x280000401:0x7da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.954609:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000a9bd3902/0xd5401cbabc0216fb lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1df9 expref: 328 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.954625:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000a9bd3902/0xd5401cbabc0216fb lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1df9 expref: 328 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.954637:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515302a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0000_UUID:9151:1870152992967808:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520930.954708:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af19c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520930.954788:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35342 00000100:00100000:3.0:1783520930.954804:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cba9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+330:2389:x1870153002305024:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.954815:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.954821:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000001af9fc5e/0xd5401cbabc0216df lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7d6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1dc1 expref: 330 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.954839:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000001af9fc5e/0xd5401cbabc0216df lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7d6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1dc1 expref: 330 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.954862:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000001af9fc5e/0xd5401cbabc0216df lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7d6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1dc1 expref: 331 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.954887:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7d6:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.954891:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7d6:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.954896:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000a2269f0c/0xd5401cbabc0243ef lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.954911:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000a2269f0c/0xd5401cbabc0243ef lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.954934:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.954962:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cba9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+332:2389:x1870153002305024:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 162us (321us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.954981:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35342 00000100:00100000:3.0:1783520930.955013:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35345 00000100:00100000:3.0:1783520930.955027:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515306680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+331:2388:x1870153002304896:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.955041:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.955051:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000001981ca1a/0xd5401cbabc0216d8 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7d5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1db3 expref: 331 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.955065:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000001981ca1a/0xd5401cbabc0216d8 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7d5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1db3 expref: 330 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.955101:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000001981ca1a/0xd5401cbabc0216d8 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7d5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1db3 expref: 329 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.955120:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7d5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.955129:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7d5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.955136:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc0243e8 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.955145:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc0243e8 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.955197:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.955224:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515306680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+326:2388:x1870153002304896:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 199us (400us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.955234:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35345 00010000:00010000:3.0:1783520930.955407:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.955445:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515305c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+324:2388:x1870153002305280:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 222us (585us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.955459:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35346 00010000:00010000:3.0:1783520930.955560:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.955570:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc0243e1 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.955608:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.955618:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc0243e1 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.955647:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc0243e1 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.955699:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc0243e1 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.955716:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc0243e1 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.955788:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.955794:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000a2269f0c/0xd5401cbabc0243ef lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.955812:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.955818:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000a2269f0c/0xd5401cbabc0243ef lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.955830:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000a2269f0c/0xd5401cbabc0243ef lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.955863:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000a2269f0c/0xd5401cbabc0243ef lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.955872:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000a2269f0c/0xd5401cbabc0243ef lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.955925:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.955930:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.955938:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.955949:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.956738:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.956923:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515303480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0000_UUID:35275:1870152992967680:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520930.956946:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024404 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.957037:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002305536 00000100:00100000:3.0:1783520930.957043:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002305536 00010000:00010000:3.0:1783520930.957069:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000915ce8fd/0xd5401cbabc0216f4 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1deb expref: 324 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.957108:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35348 00000100:00100000:3.0:1783520930.957124:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c4a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+323:2389:x1870153002305536:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.957138:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.957146:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000915ce8fd/0xd5401cbabc0216f4 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7d9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1deb expref: 323 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.957165:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000915ce8fd/0xd5401cbabc0216f4 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7d9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1deb expref: 323 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.957190:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000915ce8fd/0xd5401cbabc0216f4 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7d9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1deb expref: 323 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.957200:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7d9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.957247:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7d9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.957255:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024404 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.957276:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024404 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.957321:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.957354:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c4a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+322:2389:x1870153002305536:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 234us (434us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.957390:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35348 00010000:00010000:3.0:1783520930.957422:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024404 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.957432:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024404 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.957453:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024404 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.957467:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024404 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.957486:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520930.957497:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.957502:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024404 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.957509:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784de00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.957514:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024404 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.957527:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024404 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.957542:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024404 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.957549:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024404 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.957572:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.957576:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.957580:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.957811:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435130e3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+712:3625:x1870152992967296:12345-0@lo:6:osp-syn-0-0.0 Request processed in 3957us (4112us total) trans 4294977233 rc 0/0 00000100:00100000:3.0:1783520930.957821:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43750 00000100:00100000:3.0:1783520930.957868:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515302a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0000_UUID:9151:1870152992967808:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520930.957888:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc02440b lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.958305:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002305664 00000100:00100000:3.0:1783520930.958311:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002305664 00010000:00010000:3.0:1783520930.958333:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000a9bd3902/0xd5401cbabc0216fb lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1df9 expref: 322 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.958492:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35350 00000100:00100000:3.0:1783520930.958504:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357fc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+323:2390:x1870153002305792:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.958515:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.958524:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000008121ac3a/0xd5401cbabc021702 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x7db:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e07 expref: 323 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.958546:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000008121ac3a/0xd5401cbabc021702 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7db:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e07 expref: 323 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.958571:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7db:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.958577:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7db:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.958587:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000005120a6/0xd5401cbabc024412 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7db:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.958620:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000005120a6/0xd5401cbabc024412 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7db:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.958658:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.958687:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357fc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+322:2390:x1870153002305792:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 186us (286us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.958704:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35350 00010000:00010000:3.0:1783520930.958737:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc02440b lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.958747:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc02440b lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.958769:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc02440b lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.958779:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc02440b lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.958796:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520930.958808:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.958812:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc02440b lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.958818:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784ca00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.958821:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc02440b lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.958827:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc02440b lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.958838:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc02440b lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.958843:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc02440b lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.958861:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.958866:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.958870:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.958969:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cba8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+712:3627:x1870152992967552:12345-0@lo:6:osp-syn-0-0.0 Request processed in 4699us (4873us total) trans 4294977234 rc 0/0 00000100:00100000:3.0:1783520930.958982:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43751 00000100:00100000:3.0:1783520930.959214:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43758 00000100:00100000:3.0:1783520930.959224:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c569f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+713:3625:x1870152992969472:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520930.959279:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2017 count 1 00010000:00010000:3.0:1783520930.959292:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc02443c lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7e1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.959308:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 000000007d964937/0xd5401cbabc021733 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x7e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e5b expref: 320 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.959315:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7e1:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520930.959319:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7e1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.959326:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc02443c lrc: 4/0,1 mode: --/PW res: [0x280000401:0x7e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.959351:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 000000007d964937/0xd5401cbabc021733 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e5b expref: 320 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.959377:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 000000007d964937/0xd5401cbabc021733 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e5b expref: 320 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.959388:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515302a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0000_UUID:9151:1870152992969728:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520930.959440:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af19c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520930.959462:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002306048 00000100:00100000:3.0:1783520930.959466:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002306048 00010000:00010000:3.0:1783520930.959492:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000c9f589ae/0xd5401cbabc021709 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e15 expref: 321 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.959520:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35351 00000100:00100000:3.0:1783520930.959531:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436342bad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+322:2387:x1870153002306048:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.959542:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.959552:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000c9f589ae/0xd5401cbabc021709 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7dc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e15 expref: 322 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.959568:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000c9f589ae/0xd5401cbabc021709 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7dc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e15 expref: 322 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.959600:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c9f589ae/0xd5401cbabc021709 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7dc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e15 expref: 323 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.959628:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7dc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.959633:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7dc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.959639:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000001e04e98c/0xd5401cbabc024419 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.959656:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000001e04e98c/0xd5401cbabc024419 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.959707:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.959748:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436342bad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+323:2387:x1870153002306048:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 216us (451us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.959810:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35351 00000100:00100000:3.0:1783520930.959835:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35354 00000100:00100000:3.0:1783520930.959852:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c56a300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+319:2387:x1870153002306304:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.959869:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.959877:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000a3608d20/0xd5401cbabc02172c lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x7e0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e4d expref: 319 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.959901:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000a3608d20/0xd5401cbabc02172c lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7e0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e4d expref: 319 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.959919:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7e0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.959924:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7e0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.959927:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000e1c39107/0xd5401cbabc024435 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7e0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.959934:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000e1c39107/0xd5401cbabc024435 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7e0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.959946:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.959969:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c56a300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+318:2387:x1870153002306304:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 120us (363us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.959993:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35354 00010000:00010000:3.0:1783520930.960108:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.960113:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 0000000001e3cafb/0xd5401cbabc024427 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.960119:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfbc00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.960122:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 0000000001e3cafb/0xd5401cbabc024427 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.960148:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 0000000001e3cafb/0xd5401cbabc024427 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.960175:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 0000000001e3cafb/0xd5401cbabc024427 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.960180:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000001e3cafb/0xd5401cbabc024427 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.960204:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.960208:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.960212:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.960397:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.960402:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000001e04e98c/0xd5401cbabc024419 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.960414:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.960421:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000001e04e98c/0xd5401cbabc024419 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.960427:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000001e04e98c/0xd5401cbabc024419 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.960438:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000001e04e98c/0xd5401cbabc024419 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.960450:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000001e04e98c/0xd5401cbabc024419 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.960477:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.960482:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.960488:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.960569:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152992970240 00000100:00100000:3.0:1783520930.960575:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152992970240 00000100:00100000:3.0:1783520930.960620:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152992969984 00000100:00100000:3.0:1783520930.960627:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152992969984 00000100:00100000:3.0:1783520930.960655:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43762 00000100:00100000:3.0:1783520930.960667:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515307100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+716:3627:x1870152992969984:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520930.960682:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2019 count 1 00010000:00010000:3.0:1783520930.960695:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024458 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7e3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.960716:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000091bd37d/0xd5401cbabc021741 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x7e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e77 expref: 316 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.960742:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7e3:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520930.960751:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7e3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.960763:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024458 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x7e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.960794:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000091bd37d/0xd5401cbabc021741 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e77 expref: 316 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.960818:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000091bd37d/0xd5401cbabc021741 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e77 expref: 316 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.960845:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515303480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0000_UUID:35275:1870152992970752:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520930.960874:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af1000 going to sleep for 11 seconds 00000100:00100000:3.0:1783520930.963096:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515302a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0000_UUID:9151:1870152992969728:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520930.963128:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc02443c lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.963168:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515303480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0000_UUID:35275:1870152992970752:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520930.963195:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024458 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.963259:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c7b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0000_UUID:8409:1870152992970368:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520930.963280:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc02444a lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.963331:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002306816 00000100:00100000:3.0:1783520930.963338:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002306816 00010000:00010000:3.0:1783520930.963390:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 0000000029334aab/0xd5401cbabc02173a lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e69 expref: 320 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.963418:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35355 00000100:00100000:3.0:1783520930.963435:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+321:2388:x1870153002306688:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.963454:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.963465:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000007d964937/0xd5401cbabc021733 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7e1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e5b expref: 321 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.963492:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000007d964937/0xd5401cbabc021733 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7e1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e5b expref: 321 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.963527:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000007d964937/0xd5401cbabc021733 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7e1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e5b expref: 321 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.963553:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7e1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.963562:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7e1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.963571:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc02443c lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.963610:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc02443c lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.963645:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.963685:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+320:2388:x1870153002306688:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 255us (506us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.963703:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35355 00000100:00100000:3.0:1783520930.963740:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35359 00000100:00100000:3.0:1783520930.963755:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+319:2388:x1870153002307072:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.963769:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.963779:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000006a31c600/0xd5401cbabc021756 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7e6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ea1 expref: 318 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.963809:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000006a31c600/0xd5401cbabc021756 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7e6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ea1 expref: 317 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.963835:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000006a31c600/0xd5401cbabc021756 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7e6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ea1 expref: 317 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.963867:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7e6:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.963876:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7e6:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.963886:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000d4a09c64/0xd5401cbabc02445f lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.963899:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000d4a09c64/0xd5401cbabc02445f lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.963952:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.963979:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+315:2388:x1870153002307072:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 226us (720us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.963990:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35359 00000100:00100000:3.0:1783520930.964015:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35361 00000100:00100000:3.0:1783520930.964030:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c4e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+314:2388:x1870153002306944:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.964045:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.964055:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000091bd37d/0xd5401cbabc021741 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7e3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e77 expref: 313 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.964078:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000091bd37d/0xd5401cbabc021741 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7e3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e77 expref: 313 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.964105:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000091bd37d/0xd5401cbabc021741 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7e3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e77 expref: 312 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.964132:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7e3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.964139:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7e3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.964148:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024458 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.964158:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024458 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.964198:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.964228:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c4e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+311:2388:x1870153002306944:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 202us (769us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.964244:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35361 00010000:00010000:3.0:1783520930.964350:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc02443c lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.964432:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc02443c lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.964448:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc02443c lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.964475:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc02443c lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.964495:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.964502:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000d4a09c64/0xd5401cbabc02445f lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.964510:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.964514:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000d4a09c64/0xd5401cbabc02445f lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.964525:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000d4a09c64/0xd5401cbabc02445f lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.964551:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000d4a09c64/0xd5401cbabc02445f lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.964577:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000d4a09c64/0xd5401cbabc02445f lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.964610:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.964614:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.964616:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.964628:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520930.964698:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.964706:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc02443c lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.964715:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784c800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.964719:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc02443c lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.964727:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc02443c lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.964741:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc02443c lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.964752:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc02443c lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.964773:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.964776:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.964779:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.964872:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c569f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+721:3625:x1870152992969472:12345-0@lo:6:osp-syn-0-0.0 Request processed in 5648us (6113us total) trans 4294977242 rc 0/0 00000100:00100000:3.0:1783520930.964892:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43758 00010000:00010000:3.0:1783520930.964920:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc02444a lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.964937:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc02444a lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.964949:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc02444a lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.964974:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc02444a lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.965008:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520930.965075:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350ade2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+721:3625:x1870152992970112:12345-0@lo:6:osp-syn-0-0.0 Request processed in 4595us (4665us total) trans 4294977243 rc 0/0 00000100:00100000:3.0:1783520930.965088:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43760 00010000:00010000:3.0:1783520930.965118:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024458 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.965142:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024458 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.965155:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024458 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.965175:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024458 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.965191:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520930.965201:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.965206:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024458 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.965217:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784ce00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.965221:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024458 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.965234:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024458 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.965257:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024458 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.965267:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024458 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.965289:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.965293:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.965295:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.965466:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515307100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+723:3627:x1870152992969984:12345-0@lo:6:osp-syn-0-0.0 Request processed in 4802us (4888us total) trans 4294977245 rc 0/0 00000100:00100000:3.0:1783520930.965476:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43762 00000100:00100000:3.0:1783520930.965518:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152992972032 00000100:00100000:3.0:1783520930.965524:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152992972032 00000100:00100000:3.0:1783520930.965554:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43769 00000100:00100000:3.0:1783520930.965569:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94360550c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+723:3626:x1870152992972032:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520930.965602:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2028 count 1 00010000:00010000:3.0:1783520930.965621:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024489 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7ec:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.965646:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000e6d0cfb4/0xd5401cbabc021780 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x7ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ef5 expref: 308 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.965653:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7ec:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520930.965656:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7ec:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.965659:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024489 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x7ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.965680:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000e6d0cfb4/0xd5401cbabc021780 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ef5 expref: 308 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.965704:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000e6d0cfb4/0xd5401cbabc021780 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ef5 expref: 308 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.965734:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515303480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0000_UUID:8409:1870152992972544:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520930.965762:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af19c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520930.965809:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c6a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+308:2388:x1870153002306432:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 2344us (2666us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.965959:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35356 00010000:00010000:3.0:1783520930.966128:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc024443 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.966135:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc024443 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.966142:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc024443 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.966147:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc024443 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.966159:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520930.966166:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.966173:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc024443 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.966179:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.966183:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc024443 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.966197:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc024443 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.966213:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc024443 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.966222:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc024443 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.966239:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.966242:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.966244:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.966434:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516359f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+725:3628:x1870152992969600:12345-0@lo:6:osp-syn-0-0.0 Request processed in 7201us (7343us total) trans 4294977246 rc 0/0 00000100:00100000:3.0:1783520930.966444:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43759 00010000:00010000:3.0:1783520930.966475:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024420 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.966485:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024420 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.966493:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024420 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.966506:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024420 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.966519:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520930.966601:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+726:3626:x1870152992968320:12345-0@lo:6:osp-syn-0-0.0 Request processed in 10063us (10183us total) trans 4294977247 rc 0/0 00000100:00100000:3.0:1783520930.966626:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43754 00000004:00080000:3.0:1783520930.966678:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e1c4700/2, rc 0, transno 4294977247 00000100:00100000:3.0:1783520930.966692:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152992968320:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520930.967683:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515303480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0000_UUID:8409:1870152992972544:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520930.967707:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024489 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.967903:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002308096 00000100:00100000:3.0:1783520930.967910:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002308096 00010000:00010000:3.0:1783520930.967976:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000000400e40f/0xd5401cbabc021779 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ee7 expref: 307 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.968020:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002307584 00000100:00100000:3.0:1783520930.968026:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002307584 00010000:00010000:3.0:1783520930.968051:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000fd842d33/0xd5401cbabc02176b lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ecb expref: 309 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.968079:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35365 00000100:00100000:3.0:1783520930.968096:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e472a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+310:2389:x1870153002308096:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.968107:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.968116:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000000400e40f/0xd5401cbabc021779 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7eb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ee7 expref: 310 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.968140:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000000400e40f/0xd5401cbabc021779 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7eb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ee7 expref: 310 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.968161:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000000400e40f/0xd5401cbabc021779 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7eb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ee7 expref: 310 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.968188:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7eb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.968238:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7eb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.968250:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc024482 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.968279:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc024482 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.968328:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.968414:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e472a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+306:2389:x1870153002308096:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 266us (472us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.968437:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35365 00010000:00010000:3.0:1783520930.968467:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024489 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.968496:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024489 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.968525:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024489 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.968555:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024489 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.968606:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520930.968627:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520930.968636:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024489 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.968646:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784c200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.968652:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024489 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.968672:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024489 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.968688:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024489 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.968712:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024489 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.968747:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.968755:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc024474 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.968918:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.968925:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc024474 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.968937:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc024474 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.968953:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc024474 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.969070:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc024474 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.969093:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.969134:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.969139:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.969149:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.969167:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152992973568 00000100:00100000:3.0:1783520930.969173:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152992973568 00000100:00100000:3.0:1783520930.969220:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43774 00000100:00100000:3.0:1783520930.969235:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e473800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+729:3625:x1870152992973568:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520930.969290:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2033 count 1 00010000:00010000:3.0:1783520930.969304:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0244b3 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7f1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.969326:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 000000007c2c765d/0xd5401cbabc0217aa lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x7f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f3b expref: 300 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.969351:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7f1:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520930.969382:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7f1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.969389:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0244b3 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x7f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.969434:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 000000007c2c765d/0xd5401cbabc0217aa lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f3b expref: 300 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.969448:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 000000007c2c765d/0xd5401cbabc0217aa lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f3b expref: 300 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.969460:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515303480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0000_UUID:9151:1870152992974080:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520930.969496:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af19c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520930.969619:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94360550c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+731:3626:x1870152992972032:12345-0@lo:6:osp-syn-0-0.0 Request processed in 4051us (4125us total) trans 4294977253 rc 0/0 00000100:00100000:3.0:1783520930.969637:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43769 00000100:00100000:3.0:1783520930.969662:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152992974208 00000100:00100000:3.0:1783520930.969668:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152992974208 00000100:00100000:3.0:1783520930.969765:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152992974464 00000100:00100000:3.0:1783520930.969770:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152992974464 00000100:00100000:3.0:1783520930.969799:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43778 00000100:00100000:3.0:1783520930.969815:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e46d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+732:3626:x1870152992974464:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520930.969837:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2037 count 1 00010000:00010000:3.0:1783520930.969856:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc0244c8 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7f5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.969906:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000d69176f5/0xd5401cbabc0217c6 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x7f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f73 expref: 299 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.969935:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7f5:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520930.969946:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7f5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.969954:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc0244c8 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x7f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.970001:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000d69176f5/0xd5401cbabc0217c6 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f73 expref: 299 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.970029:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000d69176f5/0xd5401cbabc0217c6 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f73 expref: 298 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.970042:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515302a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0000_UUID:8409:1870152992974720:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520930.970081:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af10c0 going to sleep for 11 seconds 00010000:00010000:3.0:1783520930.970166:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.970172:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000a2269f0c/0xd5401cbabc0244a5 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.970181:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.970185:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000a2269f0c/0xd5401cbabc0244a5 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.970198:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000a2269f0c/0xd5401cbabc0244a5 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.970255:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000a2269f0c/0xd5401cbabc0244a5 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.970270:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000a2269f0c/0xd5401cbabc0244a5 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.970289:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.970292:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.970296:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.970425:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 0000000066426cc1/0xd5401cbabc02449e lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.970443:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 0000000066426cc1/0xd5401cbabc02449e lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.970454:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000066426cc1/0xd5401cbabc02449e lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.970474:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 0000000066426cc1/0xd5401cbabc02449e lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.970497:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520930.970509:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.970517:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 0000000066426cc1/0xd5401cbabc02449e lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.970540:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.970545:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 0000000066426cc1/0xd5401cbabc02449e lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.970572:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 0000000066426cc1/0xd5401cbabc02449e lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.970611:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 0000000066426cc1/0xd5401cbabc02449e lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.970632:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000066426cc1/0xd5401cbabc02449e lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.970667:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.970671:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.970675:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.970935:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e46e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+736:3628:x1870152992972928:12345-0@lo:6:osp-syn-0-0.0 Request processed in 4063us (4166us total) trans 4294977256 rc 0/0 00000100:00100000:3.0:1783520930.970947:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43772 00000100:00100000:3.0:1783520930.971267:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515303480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0000_UUID:9151:1870152992974080:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520930.971285:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0244b3 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.971330:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515302a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0000_UUID:8409:1870152992974720:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520930.971344:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc0244c8 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.972431:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002308864 00000100:00100000:3.0:1783520930.972436:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002308864 00010000:00010000:3.0:1783520930.972458:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000007c2c765d/0xd5401cbabc0217aa lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f3b expref: 301 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.972492:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002308992 00000100:00100000:3.0:1783520930.972497:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002308992 00010000:00010000:3.0:1783520930.972514:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000bb48259d/0xd5401cbabc0217b8 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f57 expref: 303 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.972552:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35374 00000100:00100000:3.0:1783520930.972566:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e471c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+303:2387:x1870153002309120:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.972580:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.972600:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000001fca846d/0xd5401cbabc0217bf lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7f4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f65 expref: 303 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.972698:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000001fca846d/0xd5401cbabc0217bf lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7f4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f65 expref: 301 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.972798:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000001fca846d/0xd5401cbabc0217bf lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7f4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f65 expref: 299 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.972812:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7f4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.972815:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7f4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.972821:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000005f11335d/0xd5401cbabc0244c1 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.972846:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000005f11335d/0xd5401cbabc0244c1 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.972881:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.972904:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e471c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+295:2387:x1870153002309120:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 343us (544us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.972918:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35374 00000100:00100000:3.0:1783520930.973057:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.973101:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.973178:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002309376 00000100:00100000:3.0:1783520930.973184:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002309376 00010000:00010000:3.0:1783520930.973200:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000682eb425/0xd5401cbabc0217d4 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f8f expref: 295 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.973279:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35377 00000100:00100000:3.0:1783520930.973294:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357dc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+296:2387:x1870153002309248:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.973309:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.973318:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000b70a9216/0xd5401cbabc0217cd lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7f6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f81 expref: 296 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.973428:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000b70a9216/0xd5401cbabc0217cd lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7f6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f81 expref: 296 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.973449:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000b70a9216/0xd5401cbabc0217cd lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7f6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f81 expref: 296 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.973459:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7f6:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.973466:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7f6:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.973475:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc0244cf lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.973490:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc0244cf lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.973512:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.973540:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357dc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+293:2387:x1870153002309248:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 250us (439us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.973552:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35377 00010000:00010000:3.0:1783520930.973664:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.973669:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000c42e258a/0xd5401cbabc0244dd lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.973678:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.973684:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000c42e258a/0xd5401cbabc0244dd lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.973718:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000c42e258a/0xd5401cbabc0244dd lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.973748:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000c42e258a/0xd5401cbabc0244dd lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.973768:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c42e258a/0xd5401cbabc0244dd lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.973787:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.973792:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000c8b052a2/0xd5401cbabc0244d6 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.973799:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.973805:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000c8b052a2/0xd5401cbabc0244d6 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.973823:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000c8b052a2/0xd5401cbabc0244d6 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.973838:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000c8b052a2/0xd5401cbabc0244d6 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.973859:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c8b052a2/0xd5401cbabc0244d6 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.973890:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.973893:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.973897:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.973905:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.974419:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc0244ba lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.974431:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc0244ba lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.974442:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc0244ba lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.974458:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc0244ba lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.974490:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520930.974502:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.974509:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc0244ba lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.974516:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.974519:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc0244ba lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.974537:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc0244ba lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.974552:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc0244ba lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.974566:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc0244ba lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.974615:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.974623:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc0244cf lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.974645:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.974653:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc0244cf lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.974672:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc0244cf lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.974700:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc0244cf lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.974723:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc0244cf lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.974755:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.974760:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.975025:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.975035:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.975295:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363566f850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+743:3627:x1870152992973824:12345-0@lo:6:osp-syn-0-0.0 Request processed in 5746us (5969us total) trans 4294977264 rc 0/0 00000100:00100000:3.0:1783520930.975304:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43776 00000100:00100000:3.0:1783520930.976421:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e473100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0000_UUID:8409:1870152992976768:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520930.976441:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 00000000855d7374/0xd5401cbabc0244f9 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x7fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.976607:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.976636:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002310272 00000100:00100000:3.0:1783520930.976642:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002310272 00010000:00010000:3.0:1783520930.976749:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000cd157f9f/0xd5401cbabc021813 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fff expref: 291 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.976783:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35380 00000100:00100000:3.0:1783520930.976796:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515303480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+291:2390:x1870153002310272:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.976897:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.976910:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000cd157f9f/0xd5401cbabc021813 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7ff:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fff expref: 293 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.976940:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000cd157f9f/0xd5401cbabc021813 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7ff:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fff expref: 293 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.977030:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000cd157f9f/0xd5401cbabc021813 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7ff:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fff expref: 291 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.977072:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7ff:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.977091:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7ff:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.977102:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 0000000000a37bb7/0xd5401cbabc02450e lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.977123:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000000a37bb7/0xd5401cbabc02450e lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.977161:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.977190:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515303480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+289:2390:x1870153002310272:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 394us (581us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.977264:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35380 00000100:00100000:3.0:1783520930.977449:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.977521:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520930.977567:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.977575:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 0000000000a37bb7/0xd5401cbabc02450e lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.977585:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.977606:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 0000000000a37bb7/0xd5401cbabc02450e lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.977617:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 0000000000a37bb7/0xd5401cbabc02450e lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.977639:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 0000000000a37bb7/0xd5401cbabc02450e lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.977649:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000000a37bb7/0xd5401cbabc02450e lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.977672:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.977677:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.977682:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.977697:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002309760 00000100:00100000:3.0:1783520930.977705:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002309760 00010000:00010000:3.0:1783520930.977730:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000d1b33466/0xd5401cbabc0217e2 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fab expref: 289 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.977771:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35384 00000100:00100000:3.0:1783520930.977790:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515301180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+289:2388:x1870153002309760:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.977810:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.977821:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000d1b33466/0xd5401cbabc0217e2 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x7f9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fab expref: 289 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.977857:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000d1b33466/0xd5401cbabc0217e2 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x7f9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fab expref: 289 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.977896:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000d1b33466/0xd5401cbabc0217e2 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x7f9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fab expref: 288 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.977944:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7f9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.977952:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x7f9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.977964:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc0244e4 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x7f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.977983:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc0244e4 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x7f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.978025:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.978056:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515301180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+286:2388:x1870153002309760:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 270us (539us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.978085:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35384 00010000:00010000:3.0:1783520930.978123:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.978130:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc0244e4 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.978139:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.978145:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc0244e4 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.978156:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc0244e4 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.978169:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc0244e4 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.978188:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc0244e4 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.978211:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.978216:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.978221:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.978259:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.978319:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.978520:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.978768:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35387 00000100:00100000:3.0:1783520930.978785:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515302680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+286:2387:x1870153002310528:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.978838:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.978850:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000d22c69c8/0xd5401cbabc02181a lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x800:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f200d expref: 286 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.978876:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000d22c69c8/0xd5401cbabc02181a lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x800:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f200d expref: 285 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.978908:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x800:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.978917:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x800:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.978922:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000a5d93cec/0xd5401cbabc024515 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x800:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.978937:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000a5d93cec/0xd5401cbabc024515 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x800:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.978962:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.978991:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515302680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+284:2387:x1870153002310528:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 208us (474us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.979009:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35387 00010000:00010000:3.0:1783520930.979337:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.979346:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000a5d93cec/0xd5401cbabc024515 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x800:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.979384:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.979391:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000a5d93cec/0xd5401cbabc024515 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x800:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.979416:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000a5d93cec/0xd5401cbabc024515 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x800:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.979441:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000a5d93cec/0xd5401cbabc024515 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x800:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.979453:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000a5d93cec/0xd5401cbabc024515 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x800:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.979474:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.979480:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000e3859524/0xd5401cbabc024500 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.979496:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.979501:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000e3859524/0xd5401cbabc024500 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x7fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.979518:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000e3859524/0xd5401cbabc024500 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x7fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.979547:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000e3859524/0xd5401cbabc024500 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x7fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.979562:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000e3859524/0xd5401cbabc024500 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x7fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.979586:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.979606:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.979611:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.979623:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.980955:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.981040:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.981289:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.981476:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520930.981943:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.981949:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000005120a6/0xd5401cbabc024531 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x804:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.981970:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.981976:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000005120a6/0xd5401cbabc024531 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x804:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.981987:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000005120a6/0xd5401cbabc024531 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x804:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.982006:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000005120a6/0xd5401cbabc024531 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x804:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.982020:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000005120a6/0xd5401cbabc024531 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x804:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.982229:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.982234:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc02452a lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x803:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.982240:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.982243:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc02452a lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x803:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.982248:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc02452a lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x803:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.982259:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc02452a lrc: 1/0,0 mode: --/PW res: [0x280000401:0x803:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.982263:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc02452a lrc: 0/0,0 mode: --/PW res: [0x280000401:0x803:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.982279:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.982282:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.982286:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.982296:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.982424:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002311296 00000100:00100000:3.0:1783520930.982431:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002311296 00010000:00010000:3.0:1783520930.982497:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000cd369d96/0xd5401cbabc021852 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x807:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f206f expref: 281 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.982535:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35393 00000100:00100000:3.0:1783520930.982553:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e471180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+282:2387:x1870153002311168:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.982570:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.982604:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000d3fdc87c/0xd5401cbabc02183d lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x806:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2061 expref: 282 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.982635:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000d3fdc87c/0xd5401cbabc02183d lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x806:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2061 expref: 282 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.982671:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000d3fdc87c/0xd5401cbabc02183d lrc: 0/0,0 mode: --/PR res: [0x280000401:0x806:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2061 expref: 282 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.982705:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x806:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.982710:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x806:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.982721:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 0000000001e3cafb/0xd5401cbabc02453f lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x806:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.982752:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000001e3cafb/0xd5401cbabc02453f lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x806:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.982783:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.982813:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e471180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+279:2387:x1870153002311168:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 263us (406us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.982834:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35393 00010000:00010000:3.0:1783520930.983565:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.983572:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000005f11335d/0xd5401cbabc024538 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x805:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.983587:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.983602:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000005f11335d/0xd5401cbabc024538 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x805:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.983620:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000005f11335d/0xd5401cbabc024538 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x805:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.983636:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000005f11335d/0xd5401cbabc024538 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x805:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.983663:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000005f11335d/0xd5401cbabc024538 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x805:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.983695:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.983701:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.983705:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.983783:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.983790:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc02454d lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x808:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.983808:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.983813:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc02454d lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x808:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.983835:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc02454d lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x808:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.983853:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc02454d lrc: 1/0,0 mode: --/PW res: [0x280000401:0x808:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.984003:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc02454d lrc: 0/0,0 mode: --/PW res: [0x280000401:0x808:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.984026:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.984031:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.984036:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.984051:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002311680 00000100:00100000:3.0:1783520930.984057:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002311680 00010000:00010000:3.0:1783520930.984081:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000006191a5a4/0xd5401cbabc021875 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x809:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f208b expref: 275 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.984114:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35396 00000100:00100000:3.0:1783520930.984131:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e470a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+275:2387:x1870153002311680:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.984280:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.984288:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000006191a5a4/0xd5401cbabc021875 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x809:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f208b expref: 275 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.984297:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000006191a5a4/0xd5401cbabc021875 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x809:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f208b expref: 275 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.984315:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000006191a5a4/0xd5401cbabc021875 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x809:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f208b expref: 275 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.984339:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x809:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.984344:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x809:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.984350:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000dc1a697e/0xd5401cbabc024554 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x809:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.984386:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000dc1a697e/0xd5401cbabc024554 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x809:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.984408:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.984437:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e470a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+274:2387:x1870153002311680:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 309us (411us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.984662:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35396 00000100:00100000:3.0:1783520930.984684:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002311808 00000100:00100000:3.0:1783520930.984691:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002311808 00010000:00010000:3.0:1783520930.984756:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000000eb8c385/0xd5401cbabc021860 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x80a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2099 expref: 274 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.984791:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35397 00000100:00100000:3.0:1783520930.984804:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e470380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+274:2390:x1870153002311808:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.984815:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.984825:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000000eb8c385/0xd5401cbabc021860 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x80a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2099 expref: 274 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.984842:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000000eb8c385/0xd5401cbabc021860 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x80a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2099 expref: 274 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.984997:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000000eb8c385/0xd5401cbabc021860 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x80a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2099 expref: 275 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.985028:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x80a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.985038:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x80a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.985049:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000f07fd018/0xd5401cbabc024562 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x80a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.985060:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000f07fd018/0xd5401cbabc024562 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x80a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.985092:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.985120:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e470380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+273:2390:x1870153002311808:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 319us (809us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.985128:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35397 00010000:00010000:3.0:1783520930.985352:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.985497:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000f07fd018/0xd5401cbabc024562 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x80a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.985508:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.985513:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000f07fd018/0xd5401cbabc024562 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x80a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.985530:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000f07fd018/0xd5401cbabc024562 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x80a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.985550:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000f07fd018/0xd5401cbabc024562 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x80a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.985566:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000f07fd018/0xd5401cbabc024562 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x80a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.985601:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.985703:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.985707:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.985759:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.986013:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.986330:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520930.986454:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.986462:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000c721a958/0xd5401cbabc024570 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x80d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.986479:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfa000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.986486:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000c721a958/0xd5401cbabc024570 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x80d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.986508:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000c721a958/0xd5401cbabc024570 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x80d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.986535:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000c721a958/0xd5401cbabc024570 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x80d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.986552:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c721a958/0xd5401cbabc024570 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x80d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.986604:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.986607:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.986612:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.987289:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35403 00000100:00100000:3.0:1783520930.987306:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503849f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+270:2390:x1870153002312448:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.987325:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.987337:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000001938ffcd/0xd5401cbabc02188a lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x80f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20df expref: 270 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.987407:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000001938ffcd/0xd5401cbabc02188a lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x80f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20df expref: 270 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.987446:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000001938ffcd/0xd5401cbabc02188a lrc: 0/0,0 mode: --/PR res: [0x280000401:0x80f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20df expref: 270 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.987502:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x80f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.987511:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x80f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.987522:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc02457e lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x80f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.987532:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc02457e lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x80f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.987552:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.987561:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000c42e258a/0xd5401cbabc024577 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x80e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.987578:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.987584:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000c42e258a/0xd5401cbabc024577 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x80e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.987726:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000c42e258a/0xd5401cbabc024577 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x80e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.987759:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000c42e258a/0xd5401cbabc024577 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x80e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.987771:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c42e258a/0xd5401cbabc024577 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x80e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.987845:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.987849:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.987854:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.987879:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520930.987893:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.987901:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 0000000066426cc1/0xd5401cbabc02458c lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x811:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.987915:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.987920:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 0000000066426cc1/0xd5401cbabc02458c lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x811:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.987934:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 0000000066426cc1/0xd5401cbabc02458c lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x811:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.987967:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 0000000066426cc1/0xd5401cbabc02458c lrc: 1/0,0 mode: --/PW res: [0x280000401:0x811:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.987995:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000066426cc1/0xd5401cbabc02458c lrc: 0/0,0 mode: --/PW res: [0x280000401:0x811:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.988029:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.988036:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.988041:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.988071:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503849f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+265:2390:x1870153002312448:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 771us (942us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.988103:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35403 00000100:00100000:3.0:1783520930.988237:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520930.988979:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.988987:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc02457e lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x80f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.988998:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.989002:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc02457e lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x80f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.989012:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc02457e lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x80f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.989026:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc02457e lrc: 1/0,0 mode: --/PW res: [0x280000401:0x80f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.989036:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc02457e lrc: 0/0,0 mode: --/PW res: [0x280000401:0x80f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.989057:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.989062:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.989067:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.990534:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002313344 00000100:00100000:3.0:1783520930.990540:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002313344 00010000:00010000:3.0:1783520930.990564:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000002eaa3702/0xd5401cbabc0218ad lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x814:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2125 expref: 266 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.990606:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002313600 00000100:00100000:3.0:1783520930.990614:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002313600 00010000:00010000:3.0:1783520930.990633:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000003d1b25c3/0xd5401cbabc0218c9 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x819:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f216b expref: 268 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.990671:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35409 00000100:00100000:3.0:1783520930.990687:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e47b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+268:2388:x1870153002313344:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.990703:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.990714:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000002eaa3702/0xd5401cbabc0218ad lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x814:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2125 expref: 268 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.990744:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000002eaa3702/0xd5401cbabc0218ad lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x814:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2125 expref: 268 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.990793:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000002eaa3702/0xd5401cbabc0218ad lrc: 0/0,0 mode: --/PR res: [0x280000401:0x814:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2125 expref: 266 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.990819:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x814:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.990827:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x814:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.990836:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc0245b6 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x814:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.990845:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc0245b6 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x814:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.990873:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.990900:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e47b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+264:2388:x1870153002313344:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 216us (442us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.990915:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35409 00000100:00100000:3.0:1783520930.990941:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35411 00000100:00100000:3.0:1783520930.990955:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e479f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+263:2389:x1870153002313600:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.990971:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.990995:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000003d1b25c3/0xd5401cbabc0218c9 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x819:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f216b expref: 261 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.991019:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000003d1b25c3/0xd5401cbabc0218c9 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x819:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f216b expref: 261 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.991053:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x819:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.991061:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x819:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.991072:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000c1a73293/0xd5401cbabc0245c4 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x819:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.991085:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000c1a73293/0xd5401cbabc0245c4 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x819:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.991101:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520930.991128:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.991137:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc0245b6 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x814:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.991156:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.991164:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc0245b6 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x814:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.991179:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc0245b6 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x814:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.991199:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc0245b6 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x814:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.991218:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc0245b6 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x814:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.991257:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.991265:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000855d7374/0xd5401cbabc0245af lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x817:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.991273:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfa400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.991280:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000855d7374/0xd5401cbabc0245af lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x817:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.991306:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000855d7374/0xd5401cbabc0245af lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x817:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.991320:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000855d7374/0xd5401cbabc0245af lrc: 1/0,0 mode: --/PW res: [0x280000401:0x817:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.991332:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000855d7374/0xd5401cbabc0245af lrc: 0/0,0 mode: --/PW res: [0x280000401:0x817:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.991349:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.991352:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.991355:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.991393:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e479f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+260:2389:x1870153002313600:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 440us (848us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.991409:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35411 00010000:00010000:3.0:1783520930.991753:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.992280:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.992318:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.992416:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.992646:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35412 00000100:00100000:3.0:1783520930.992661:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515302d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+261:2390:x1870153002313728:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.992679:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.992690:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000682c187e/0xd5401cbabc0218de lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x81b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2187 expref: 261 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.992758:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000682c187e/0xd5401cbabc0218de lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x81b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2187 expref: 261 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.992792:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x81b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.992797:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x81b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.992804:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000007b280ca1/0xd5401cbabc0245d2 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x81b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.992847:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000007b280ca1/0xd5401cbabc0245d2 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x81b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.992886:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.992919:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515302d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+259:2390:x1870153002313728:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 263us (641us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.992933:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35412 00010000:00010000:3.0:1783520930.993157:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520930.993164:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000007aa431df/0xd5401cbabc0245a8 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x815:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.993176:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.993183:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000007aa431df/0xd5401cbabc0245a8 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x815:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.993200:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000007aa431df/0xd5401cbabc0245a8 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x815:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.993219:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000007aa431df/0xd5401cbabc0245a8 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x815:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.993242:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000007aa431df/0xd5401cbabc0245a8 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x815:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.993288:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.993293:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.993407:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.993421:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.993757:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.993803:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.993890:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.993952:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.993986:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35415 00000100:00100000:3.0:1783520930.994091:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515300a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+259:2388:x1870153002314112:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.994108:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.994122:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000b2e118c0/0xd5401cbabc0218ec lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x81c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2195 expref: 259 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.994153:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000b2e118c0/0xd5401cbabc0218ec lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x81c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2195 expref: 259 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.994189:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000b2e118c0/0xd5401cbabc0218ec lrc: 0/0,0 mode: --/PR res: [0x280000401:0x81c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2195 expref: 258 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.994211:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x81c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.994216:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x81c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.994274:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000003a22489f/0xd5401cbabc0245e0 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x81c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.994292:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000003a22489f/0xd5401cbabc0245e0 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x81c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.994458:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.994526:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515300a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+256:2388:x1870153002314112:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 526us (771us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.994547:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35415 00000100:00100000:3.0:1783520930.994574:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35419 00000100:00100000:3.0:1783520930.994584:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c552a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+255:2390:x1870153002314624:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.994610:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.994621:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000c2c40b93/0xd5401cbabc021901 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x820:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21cd expref: 255 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.994636:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000c2c40b93/0xd5401cbabc021901 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x820:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21cd expref: 254 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.994654:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x820:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.994722:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x820:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.994736:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000be742370/0xd5401cbabc0245f5 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x820:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.994763:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000be742370/0xd5401cbabc0245f5 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x820:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.994791:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.994819:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c552a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+253:2390:x1870153002314624:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 234us (499us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.994840:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35419 00010000:00010000:3.0:1783520930.994862:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.994870:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc0245ee lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x81e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.994882:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.994889:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc0245ee lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x81e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.994901:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc0245ee lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x81e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.994923:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc0245ee lrc: 1/0,0 mode: --/PW res: [0x280000401:0x81e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.994943:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc0245ee lrc: 0/0,0 mode: --/PW res: [0x280000401:0x81e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.994984:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.994988:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.994994:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.995203:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.995210:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000e1c39107/0xd5401cbabc0245d9 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x81d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.995222:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfa600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.995230:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000e1c39107/0xd5401cbabc0245d9 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x81d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.995252:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000e1c39107/0xd5401cbabc0245d9 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x81d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.995280:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000e1c39107/0xd5401cbabc0245d9 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x81d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.995300:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000e1c39107/0xd5401cbabc0245d9 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x81d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.995334:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.995341:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc0245e7 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x81f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.995354:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.995390:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc0245e7 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x81f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.995407:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc0245e7 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x81f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.995522:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc0245e7 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x81f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.995534:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc0245e7 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x81f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.995564:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.995568:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.995574:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.995656:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.995726:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520930.995974:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35421 00000100:00100000:3.0:1783520930.995991:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515300e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+252:2387:x1870153002314752:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.996002:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.996011:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000002c84cc50/0xd5401cbabc021908 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x821:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21db expref: 251 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.996040:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000002c84cc50/0xd5401cbabc021908 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x821:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21db expref: 251 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.996058:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000002c84cc50/0xd5401cbabc021908 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x821:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21db expref: 251 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.996135:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x821:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.996142:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x821:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.996197:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc0245fc lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x821:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.996213:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc0245fc lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x821:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.996240:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.996245:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000001e04e98c/0xd5401cbabc024603 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x822:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.996256:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.996262:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000001e04e98c/0xd5401cbabc024603 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x822:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.996278:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000001e04e98c/0xd5401cbabc024603 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x822:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.996295:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000001e04e98c/0xd5401cbabc024603 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x822:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.996312:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000001e04e98c/0xd5401cbabc024603 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x822:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.996420:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.996425:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.996429:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.997833:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.997838:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc024618 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x825:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.997846:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.997851:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc024618 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x825:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.997861:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc024618 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x825:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.997976:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc024618 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x825:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.997984:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc024618 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x825:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.997997:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.997998:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.998000:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.998089:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520930.998093:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024611 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x824:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.998101:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.998104:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024611 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x824:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.998113:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024611 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x824:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.998126:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024611 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x824:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.998134:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024611 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x824:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.998262:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.998265:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.998268:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.998466:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002315392 00000100:00100000:3.0:1783520930.998509:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002315392 00010000:00010000:3.0:1783520930.998535:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 0000000046245395/0xd5401cbabc02192b lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x826:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2221 expref: 246 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.998567:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002315520 00000100:00100000:3.0:1783520930.998572:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002315520 00010000:00010000:3.0:1783520930.998666:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000e15180f7/0xd5401cbabc021932 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x827:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f222f expref: 249 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520930.998699:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35427 00000100:00100000:3.0:1783520930.998710:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c551180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+249:2390:x1870153002315648:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520930.998719:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520930.998726:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000b184b81d/0xd5401cbabc021940 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x828:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f223d expref: 249 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.998747:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000b184b81d/0xd5401cbabc021940 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x828:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f223d expref: 249 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.998778:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x828:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.998782:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x828:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520930.998793:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000dc1a697e/0xd5401cbabc02462d lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x828:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.998808:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000dc1a697e/0xd5401cbabc02462d lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x828:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.998902:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520930.998929:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c551180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+247:2390:x1870153002315648:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 222us (305us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520930.998962:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35427 00010000:00010000:3.0:1783520930.999008:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 00000000dc1a697e/0xd5401cbabc02462d lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x828:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.999029:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 00000000dc1a697e/0xd5401cbabc02462d lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x828:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.999114:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000dc1a697e/0xd5401cbabc02462d lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x828:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.999143:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 00000000dc1a697e/0xd5401cbabc02462d lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x828:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.999163:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520930.999257:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+789:3626:x1870152992987520:12345-0@lo:6:osp-syn-0-0.0 Request processed in 2958us (3344us total) trans 4294977310 rc 0/0 00000100:00100000:3.0:1783520930.999275:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43829 00000004:00080000:3.0:1783520930.999419:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e471500/2, rc 0, transno 4294977310 00000100:00100000:3.0:1783520930.999439:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e471500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152992987520:0@lo:6:osp-syn-0-0.0 00010000:00010000:3.0:1783520930.999579:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520930.999585:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc024634 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x829:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.999605:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.999609:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc024634 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x829:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.999618:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc024634 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x829:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.999630:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc024634 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x829:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.999638:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc024634 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x829:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.999699:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520930.999744:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 0000000018903c53/0xd5401cbabc02461f lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x826:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.999755:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520930.999763:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 0000000018903c53/0xd5401cbabc02461f lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x826:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.999782:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 0000000018903c53/0xd5401cbabc02461f lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x826:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.999801:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 0000000018903c53/0xd5401cbabc02461f lrc: 1/0,0 mode: --/PW res: [0x280000401:0x826:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.999812:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000018903c53/0xd5401cbabc02461f lrc: 0/0,0 mode: --/PW res: [0x280000401:0x826:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520930.999833:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.999836:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.999839:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520930.999849:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520930.999857:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1783520930.999868:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c551f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152992989440:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520930.999900:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520930.999951:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.000031:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.000053:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002316160 00000100:00100000:3.0:1783520931.000059:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002316160 00010000:00010000:3.0:1783520931.000119:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000000ef9e284/0xd5401cbabc021963 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x82c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2275 expref: 241 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.000142:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002316288 00000100:00100000:3.0:1783520931.000149:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002316288 00010000:00010000:3.0:1783520931.000169:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000427c6697/0xd5401cbabc02196a lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x82d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2283 expref: 242 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.000194:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35432 00000100:00100000:3.0:1783520931.000206:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515301c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+242:2387:x1870153002316288:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.000215:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.000219:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000427c6697/0xd5401cbabc02196a lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x82d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2283 expref: 242 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.000237:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000427c6697/0xd5401cbabc02196a lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x82d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2283 expref: 241 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.000254:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000427c6697/0xd5401cbabc02196a lrc: 0/0,0 mode: --/PR res: [0x280000401:0x82d:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2283 expref: 241 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.000278:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x82d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.000282:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x82d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.000286:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 0000000000a37bb7/0xd5401cbabc024650 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x82d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.000335:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000000a37bb7/0xd5401cbabc024650 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x82d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.000386:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.000412:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515301c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+239:2387:x1870153002316288:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 207us (380us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.000440:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35432 00010000:00010000:3.0:1783520931.001306:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.001312:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000005120a6/0xd5401cbabc024649 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x82c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.001321:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.001325:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000005120a6/0xd5401cbabc024649 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x82c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.001335:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000005120a6/0xd5401cbabc024649 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x82c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.001350:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000005120a6/0xd5401cbabc024649 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x82c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.001412:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000005120a6/0xd5401cbabc024649 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x82c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.001608:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.001612:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.001615:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.001916:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35434 00000100:00100000:3.0:1783520931.001932:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638584850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+241:2390:x1870153002316672:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.001950:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.002006:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000007d6b2e2e/0xd5401cbabc021978 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x82f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f229f expref: 241 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.002034:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000007d6b2e2e/0xd5401cbabc021978 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x82f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f229f expref: 240 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.002069:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000007d6b2e2e/0xd5401cbabc021978 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x82f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f229f expref: 240 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.002101:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x82f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.002111:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x82f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.002122:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000841b1b91/0xd5401cbabc024665 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x82f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.002139:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000841b1b91/0xd5401cbabc024665 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x82f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.002168:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.002255:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638584850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+235:2390:x1870153002316672:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 270us (484us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.002276:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35434 00010000:00010000:3.0:1783520931.002322:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520931.002335:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000c8b052a2/0xd5401cbabc02465e lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x830:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.002351:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.002383:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000c8b052a2/0xd5401cbabc02465e lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x830:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.002677:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000c8b052a2/0xd5401cbabc02465e lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x830:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.002701:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000c8b052a2/0xd5401cbabc02465e lrc: 1/0,0 mode: --/PW res: [0x280000401:0x830:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.002710:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c8b052a2/0xd5401cbabc02465e lrc: 0/0,0 mode: --/PW res: [0x280000401:0x830:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.002733:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.002738:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.002742:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520931.002893:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c551f80/2, rc 0, transno 4294977320 00000100:00100000:3.0:1783520931.002907:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c551f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152992989440:0@lo:6:osp-syn-0-0.0 00010000:00010000:3.0:1783520931.004541:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.004549:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.004555:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1783520931.004569:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350792d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152992992512:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.004629:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.004675:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152992992512 00000100:00100000:3.0:1783520931.004680:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152992992512 00000100:00100000:3.0:1783520931.004783:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43849 00000100:00100000:3.0:1783520931.004796:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515300000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+803:3626:x1870152992992512:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.004809:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2108 count 1 00010000:00010000:3.0:1783520931.004827:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc0246b9 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x83c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.004876:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 0000000088a9fda6/0xd5401cbabc02120f lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x83c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2355 expref: 229 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.004885:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x83c:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.004894:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x83c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.004905:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc0246b9 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x83c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.004975:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 0000000088a9fda6/0xd5401cbabc02120f lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x83c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2355 expref: 229 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.004998:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 0000000088a9fda6/0xd5401cbabc02120f lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x83c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2355 expref: 229 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.005027:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515301c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0000_UUID:9151:1870152992992896:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.005077:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af13c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520931.006546:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002317824 00000100:00100000:3.0:1783520931.006552:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002317824 00010000:00010000:3.0:1783520931.006573:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 0000000095d5ed85/0xd5401cbabc02114b lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x839:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f232b expref: 228 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.006608:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002317952 00000100:00100000:3.0:1783520931.006611:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002317952 00010000:00010000:3.0:1783520931.006689:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000f121657c/0xd5401cbabc02118a lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x83a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2339 expref: 231 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.006726:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35443 00000100:00100000:3.0:1783520931.006733:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e478700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+232:2387:x1870153002317824:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.006745:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.006754:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 0000000095d5ed85/0xd5401cbabc02114b lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x839:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f232b expref: 232 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.006765:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 0000000095d5ed85/0xd5401cbabc02114b lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x839:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f232b expref: 232 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.006785:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x839:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.006791:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x839:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.006802:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc0246a4 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x839:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.006816:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc0246a4 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x839:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.006839:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.006865:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e478700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+231:2387:x1870153002317824:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 132us (381us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.006879:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35443 00000100:00100000:3.0:1783520931.006899:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35446 00000100:00100000:3.0:1783520931.006907:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e47bb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+230:2389:x1870153002317952:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.006917:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.006923:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000f121657c/0xd5401cbabc02118a lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x83a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2339 expref: 230 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.006945:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000f121657c/0xd5401cbabc02118a lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x83a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2339 expref: 230 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.007010:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000f121657c/0xd5401cbabc02118a lrc: 0/0,0 mode: --/PR res: [0x280000401:0x83a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2339 expref: 228 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.007077:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x83a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.007082:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x83a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.007090:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc0246ab lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x83a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.007100:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc0246ab lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x83a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.007126:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.007132:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000c1a73293/0xd5401cbabc02469d lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x838:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.007146:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.007152:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000c1a73293/0xd5401cbabc02469d lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x838:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.007183:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000c1a73293/0xd5401cbabc02469d lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x838:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.007196:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000c1a73293/0xd5401cbabc02469d lrc: 1/0,0 mode: --/PW res: [0x280000401:0x838:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.007205:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c1a73293/0xd5401cbabc02469d lrc: 0/0,0 mode: --/PW res: [0x280000401:0x838:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.007222:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.007225:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc0246b2 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x83b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.007232:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.007236:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc0246b2 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x83b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.007244:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc0246b2 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x83b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.007257:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc0246b2 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x83b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.007275:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc0246b2 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x83b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.007298:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.007301:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.007305:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.007314:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.007336:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.007380:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e47bb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+225:2389:x1870153002317952:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 473us (856us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.007388:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35446 00010000:00010000:3.0:1783520931.007431:0:9151:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000088a9fda6/0xd5401cbabc02120f lrc: 0/0,0 mode: --/PR res: [0x280000401:0x83c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2355 expref: 224 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.007459:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515301c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0000_UUID:9151:1870152992992896:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.007477:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc0246b9 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x83c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.007489:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc0246b9 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x83c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.007496:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc0246b9 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x83c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.007505:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc0246b9 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x83c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.007515:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc0246b9 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x83c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.007530:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520931.007664:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515300000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+809:3626:x1870152992992512:12345-0@lo:6:osp-syn-0-0.0 Request processed in 2871us (3039us total) trans 4294977330 rc 0/0 00000100:00100000:3.0:1783520931.007681:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43849 00000100:00100000:3.0:1783520931.007702:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43854 00000100:00100000:3.0:1783520931.007715:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350792e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+808:3626:x1870152992993920:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.007733:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2113 count 1 00010000:00010000:3.0:1783520931.007748:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc0246dc lrc: 3/0,1 mode: --/PW res: [0x280000401:0x841:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.007778:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 000000005aba8529/0xd5401cbabc021415 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x841:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f239b expref: 225 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.007803:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x841:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.007812:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x841:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.007821:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc0246dc lrc: 4/0,1 mode: --/PW res: [0x280000401:0x841:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.007854:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 000000005aba8529/0xd5401cbabc021415 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x841:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f239b expref: 225 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.007872:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 000000005aba8529/0xd5401cbabc021415 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x841:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f239b expref: 224 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.007895:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515301c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0000_UUID:9151:1870152992994176:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.007923:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af1000 going to sleep for 11 seconds 00000004:00080000:3.0:1783520931.007951:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350792d880/2, rc 0, transno 4294977330 00000100:00100000:3.0:1783520931.007970:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350792d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152992992512:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.008001:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35449 00000100:00100000:3.0:1783520931.008014:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507938700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+223:2388:x1870153002318336:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.008026:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.008035:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000f2323f3c/0xd5401cbabc021335 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x83e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2371 expref: 223 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.008058:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000f2323f3c/0xd5401cbabc021335 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x83e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2371 expref: 223 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.008087:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000f2323f3c/0xd5401cbabc021335 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x83e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2371 expref: 223 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.008102:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x83e:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.008106:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x83e:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.008112:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc0246c7 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x83e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.008125:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc0246c7 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x83e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.008146:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.008153:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000d4a09c64/0xd5401cbabc0246ce lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x83f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.008166:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.008172:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000d4a09c64/0xd5401cbabc0246ce lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x83f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.008194:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000d4a09c64/0xd5401cbabc0246ce lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x83f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.008214:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000d4a09c64/0xd5401cbabc0246ce lrc: 1/0,0 mode: --/PW res: [0x280000401:0x83f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.008229:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000d4a09c64/0xd5401cbabc0246ce lrc: 0/0,0 mode: --/PW res: [0x280000401:0x83f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.008251:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.008254:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.008257:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.008274:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.008296:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507938700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+221:2388:x1870153002318336:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 285us (850us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.008305:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35449 00010000:00010000:3.0:1783520931.008327:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.008333:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc0246ab lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x83a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.008433:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.008436:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc0246ab lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x83a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.008445:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc0246ab lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x83a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.008456:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc0246ab lrc: 1/0,0 mode: --/PW res: [0x280000401:0x83a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.008473:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc0246ab lrc: 0/0,0 mode: --/PW res: [0x280000401:0x83a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.008532:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.008535:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc0246c7 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x83e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.008548:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.008553:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc0246c7 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x83e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.008685:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc0246c7 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x83e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.008706:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc0246c7 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x83e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.008715:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc0246c7 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x83e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.008729:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.008733:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.008736:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.008750:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350793ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152992994688:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.008785:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.009480:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.009542:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.009573:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515301c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0000_UUID:9151:1870152992994176:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.009671:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc0246dc lrc: 3/0,1 mode: --/PW res: [0x280000401:0x841:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.009764:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc0246dc lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x841:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.009772:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc0246dc lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x841:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.009781:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc0246dc lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x841:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.009789:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc0246dc lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x841:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.009802:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520931.009812:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.009877:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc0246dc lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x841:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.009885:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784cc00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.009888:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc0246dc lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x841:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.009897:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc0246dc lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x841:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.009907:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc0246dc lrc: 1/0,0 mode: --/PW res: [0x280000401:0x841:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.009914:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc0246dc lrc: 0/0,0 mode: --/PW res: [0x280000401:0x841:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.009963:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.009967:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.009970:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.010057:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350792e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+816:3626:x1870152992993920:12345-0@lo:6:osp-syn-0-0.0 Request processed in 2343us (2543us total) trans 4294977336 rc 0/0 00000100:00100000:3.0:1783520931.010070:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43854 00000004:00080000:3.0:1783520931.010104:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350792ed80/2, rc 0, transno 4294977336 00000100:00100000:3.0:1783520931.010119:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350792ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152992993920:0@lo:6:osp-syn-0-0.0 00000004:00080000:3.0:1783520931.010130:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350792c000/2, rc 0, transno 4294977337 00000100:00100000:3.0:1783520931.010133:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350792c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152992993792:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.010205:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350793df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152992995968:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.010235:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.010491:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520931.010790:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.010796:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000f07fd018/0xd5401cbabc0246ea lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x843:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.010804:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.010807:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000f07fd018/0xd5401cbabc0246ea lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x843:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.010816:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000f07fd018/0xd5401cbabc0246ea lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x843:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.010826:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000f07fd018/0xd5401cbabc0246ea lrc: 1/0,0 mode: --/PW res: [0x280000401:0x843:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.010833:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000f07fd018/0xd5401cbabc0246ea lrc: 0/0,0 mode: --/PW res: [0x280000401:0x843:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.010854:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.010857:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.010859:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.010878:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.010896:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002319104 00000100:00100000:3.0:1783520931.010901:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002319104 00010000:00010000:3.0:1783520931.010921:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000008be6b325/0xd5401cbabc021446 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x842:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23a9 expref: 217 pid: 35274 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.010953:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35454 00000100:00100000:3.0:1783520931.010966:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515301180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+217:2390:x1870153002319104:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.011108:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.011114:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000008be6b325/0xd5401cbabc021446 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x842:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23a9 expref: 217 pid: 35274 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.011121:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000008be6b325/0xd5401cbabc021446 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x842:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23a9 expref: 217 pid: 35274 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.011130:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000008be6b325/0xd5401cbabc021446 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x842:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23a9 expref: 217 pid: 35274 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.011137:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x842:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.011139:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x842:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.011142:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc0246e3 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x842:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.011151:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc0246e3 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x842:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.011169:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.011187:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515301180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+216:2390:x1870153002319104:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 226us (309us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.011199:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35454 00000100:00100000:3.0:1783520931.011307:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.011476:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.011562:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.011643:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.011747:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.011801:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.011861:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002319744 00000100:00100000:3.0:1783520931.011867:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002319744 00010000:00010000:3.0:1783520931.011886:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000a7dae46e/0xd5401cbabc02195c lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x849:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f240b expref: 216 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.012012:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152992996608 00000100:00100000:3.0:1783520931.012016:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152992996608 00000100:00100000:3.0:1783520931.012040:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43864 00000100:00100000:3.0:1783520931.012049:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350793d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+818:3628:x1870152992996608:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.012065:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2123 count 1 00010000:00010000:3.0:1783520931.012079:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024722 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x84b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.012126:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000406309d1/0xd5401cbabc0219a2 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x84b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2427 expref: 214 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.012142:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x84b:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.012151:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x84b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.012163:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024722 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x84b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.012204:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000406309d1/0xd5401cbabc0219a2 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x84b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2427 expref: 212 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.012218:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000406309d1/0xd5401cbabc0219a2 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x84b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2427 expref: 212 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.012247:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515303480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0000_UUID:9151:1870152992996736:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.012275:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af1000 going to sleep for 11 seconds 00010000:00010000:3.0:1783520931.012308:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.012316:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc0246f8 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x846:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.012337:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.012345:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc0246f8 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x846:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.012497:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc0246f8 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x846:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.012555:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc0246f8 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x846:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.012566:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc0246f8 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x846:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.012605:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.012655:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.012659:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.012684:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520931.012703:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350793ed80/2, rc 0, transno 4294977341 00000100:00100000:3.0:1783520931.012721:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350793ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152992994688:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.012730:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1783520931.012744:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350793e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152992996992:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.012781:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.012803:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350793e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152992996864:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.012832:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.012904:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002320000 00000100:00100000:3.0:1783520931.012909:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002320000 00010000:00010000:3.0:1783520931.012943:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000e7af8559/0xd5401cbabc021994 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x84a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2419 expref: 210 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.012987:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35461 00000100:00100000:3.0:1783520931.012996:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e47a300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+210:2388:x1870153002320000:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.013046:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.013055:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000e7af8559/0xd5401cbabc021994 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x84a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2419 expref: 210 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.013078:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000e7af8559/0xd5401cbabc021994 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x84a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2419 expref: 210 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.013100:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000e7af8559/0xd5401cbabc021994 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x84a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2419 expref: 210 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.013120:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x84a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.013127:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x84a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.013133:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc02471b lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x84a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.013157:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc02471b lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x84a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.013179:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.013206:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e47a300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+209:2388:x1870153002320000:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 209us (331us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.013215:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35461 00010000:00010000:3.0:1783520931.013475:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520931.013488:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.013494:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024714 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x849:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.013503:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.013507:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024714 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x849:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.013523:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024714 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x849:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.013549:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024714 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x849:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.013571:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024714 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x849:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.013674:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.013679:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc02471b lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x84a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.013691:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.013697:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc02471b lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x84a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.013717:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc02471b lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x84a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.013745:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc02471b lrc: 1/0,0 mode: --/PW res: [0x280000401:0x84a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.013766:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc02471b lrc: 0/0,0 mode: --/PW res: [0x280000401:0x84a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.013849:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.013853:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.013856:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.013864:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.014170:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515303480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0000_UUID:9151:1870152992996736:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.014186:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024722 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x84b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.014385:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515301c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+825:3626:x1870152992995968:12345-0@lo:6:osp-syn-0-0.0 Request processed in 4075us (4150us total) trans 4294977346 rc 0/0 00000100:00100000:3.0:1783520931.014394:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43862 00010000:00010000:3.0:1783520931.014574:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024722 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x84b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.014600:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024722 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x84b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.014611:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024722 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x84b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.014637:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024722 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x84b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.014666:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520931.014753:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350793d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+826:3628:x1870152992996608:12345-0@lo:6:osp-syn-0-0.0 Request processed in 2702us (2807us total) trans 4294977347 rc 0/0 00000100:00100000:3.0:1783520931.014814:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43864 00000100:00100000:3.0:1783520931.014909:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35464 00000100:00100000:3.0:1783520931.014918:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350793ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+209:2388:x1870153002320384:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.015007:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.015014:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000001fa51dae/0xd5401cbabc0219be lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x84d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2443 expref: 208 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.015031:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000001fa51dae/0xd5401cbabc0219be lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x84d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2443 expref: 208 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.015051:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000001fa51dae/0xd5401cbabc0219be lrc: 0/0,0 mode: --/PR res: [0x280000401:0x84d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2443 expref: 208 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.015065:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x84d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.015069:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x84d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.015076:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc024729 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x84d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.015101:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc024729 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x84d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.015125:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.015150:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350793ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+206:2388:x1870153002320384:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 231us (390us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.015168:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35464 00010000:00010000:3.0:1783520931.015384:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.015519:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152992999040 00000100:00100000:3.0:1783520931.015523:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152992999040 00000100:00100000:3.0:1783520931.015550:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43874 00000100:00100000:3.0:1783520931.015625:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e47bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+828:3626:x1870152992999040:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.015636:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2133 count 1 00010000:00010000:3.0:1783520931.015647:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024768 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x855:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.015658:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 0000000036406723/0xd5401cbabc021a43 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x855:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f24b3 expref: 205 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.015665:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x855:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.015668:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x855:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.015672:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024768 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x855:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.015759:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 0000000036406723/0xd5401cbabc021a43 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x855:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24b3 expref: 207 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.015770:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 0000000036406723/0xd5401cbabc021a43 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x855:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24b3 expref: 207 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.015783:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515300e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0000_UUID:9151:1870152992999296:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.015869:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af1600 going to sleep for 11 seconds 00000100:00100000:3.0:1783520931.015891:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35467 00000100:00100000:3.0:1783520931.015902:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c552680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+210:2390:x1870153002320768:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.015911:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.015916:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000004013ce7d/0xd5401cbabc0219f6 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x851:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f247b expref: 210 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.015939:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000004013ce7d/0xd5401cbabc0219f6 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x851:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f247b expref: 210 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.015962:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x851:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.016019:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x851:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.016024:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000855d7374/0xd5401cbabc02474c lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x851:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.016033:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000855d7374/0xd5401cbabc02474c lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x851:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.016057:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.016077:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c552680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+208:2390:x1870153002320768:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 175us (379us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.016090:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35467 00010000:00010000:3.0:1783520931.016342:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.016483:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc024745 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x850:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.016495:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.016501:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc024745 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x850:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.016527:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc024745 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x850:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.016555:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc024745 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x850:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.016571:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc024745 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x850:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.016615:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520931.016621:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000e1c39107/0xd5401cbabc02473e lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x84f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.016629:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfa600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.016638:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000e1c39107/0xd5401cbabc02473e lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x84f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.016670:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000e1c39107/0xd5401cbabc02473e lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x84f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.016696:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000e1c39107/0xd5401cbabc02473e lrc: 1/0,0 mode: --/PW res: [0x280000401:0x84f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.016718:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000e1c39107/0xd5401cbabc02473e lrc: 0/0,0 mode: --/PW res: [0x280000401:0x84f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.016744:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.016746:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.016750:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.016757:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.016788:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.016907:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.017067:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002321152 00000100:00100000:3.0:1783520931.017072:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002321152 00010000:00010000:3.0:1783520931.017092:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000071d2fd0/0xd5401cbabc021d76 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x853:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2497 expref: 201 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.017126:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002321280 00000100:00100000:3.0:1783520931.017132:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002321280 00010000:00010000:3.0:1783520931.017146:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000006266ad48/0xd5401cbabc021a19 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x854:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24a5 expref: 202 pid: 12734 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.017177:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35470 00000100:00100000:3.0:1783520931.017191:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515300000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+202:2387:x1870153002321152:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.017200:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.017205:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000071d2fd0/0xd5401cbabc021d76 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x853:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2497 expref: 202 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.017224:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000071d2fd0/0xd5401cbabc021d76 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x853:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2497 expref: 202 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.017244:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000071d2fd0/0xd5401cbabc021d76 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x853:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2497 expref: 202 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.017256:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x853:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.017263:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x853:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.017274:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc02475a lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x853:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.017295:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc02475a lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x853:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.017326:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.017350:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515300000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+201:2387:x1870153002321152:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 160us (559us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.017395:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35470 00000100:00100000:3.0:1783520931.017444:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515300e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0000_UUID:9151:1870152992999296:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.017460:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024768 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x855:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.017479:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.017487:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc02475a lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x853:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.017498:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.017502:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc02475a lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x853:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.017510:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc02475a lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x853:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.017524:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc02475a lrc: 1/0,0 mode: --/PW res: [0x280000401:0x853:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.017536:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc02475a lrc: 0/0,0 mode: --/PW res: [0x280000401:0x853:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.017560:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.017564:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.017570:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.017608:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520931.017776:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.017783:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000c3fe08ef/0xd5401cbabc024761 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x854:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.017804:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfa200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.017809:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000c3fe08ef/0xd5401cbabc024761 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x854:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.017818:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000c3fe08ef/0xd5401cbabc024761 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x854:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.017830:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000c3fe08ef/0xd5401cbabc024761 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x854:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.017840:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c3fe08ef/0xd5401cbabc024761 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x854:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.017857:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.017861:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.017862:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.018830:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35474 00000100:00100000:3.0:1783520931.018838:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350793aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+199:2390:x1870153002321664:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.018844:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.018849:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000007bef1139/0xd5401cbabc021aac lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x856:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24c1 expref: 199 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.018860:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000007bef1139/0xd5401cbabc021aac lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x856:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24c1 expref: 199 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.018878:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000007bef1139/0xd5401cbabc021aac lrc: 0/0,0 mode: --/PR res: [0x280000401:0x856:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24c1 expref: 199 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.018889:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x856:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.018892:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x856:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.018896:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 0000000066426cc1/0xd5401cbabc024776 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x856:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.018903:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000066426cc1/0xd5401cbabc024776 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x856:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.018921:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.018941:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350793aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+197:2390:x1870153002321664:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 104us (239us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.018948:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35474 00010000:00010000:3.0:1783520931.018968:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 0000000066426cc1/0xd5401cbabc024776 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x856:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.018977:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 0000000066426cc1/0xd5401cbabc024776 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x856:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.018988:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000066426cc1/0xd5401cbabc024776 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x856:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.019000:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 0000000066426cc1/0xd5401cbabc024776 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x856:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.019131:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520931.019143:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.019149:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc02476f lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x857:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.019162:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.019165:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc02476f lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x857:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.019175:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc02476f lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x857:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.019192:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc02476f lrc: 1/0,0 mode: --/PW res: [0x280000401:0x857:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.019206:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc02476f lrc: 0/0,0 mode: --/PW res: [0x280000401:0x857:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.019237:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.019241:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.019245:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.019473:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435130e1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+837:3627:x1870152992999424:12345-0@lo:6:osp-syn-0-0.0 Request processed in 2847us (2935us total) trans 4294977359 rc 0/0 00000100:00100000:3.0:1783520931.019484:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43875 00010000:00010000:3.0:1783520931.019504:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.019510:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc02477d lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x858:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.019519:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.019525:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc02477d lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x858:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.019535:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc02477d lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x858:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.019549:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc02477d lrc: 1/0,0 mode: --/PW res: [0x280000401:0x858:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.019564:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc02477d lrc: 0/0,0 mode: --/PW res: [0x280000401:0x858:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.019583:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.019599:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.019604:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.019621:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993001344 00000100:00100000:3.0:1783520931.019626:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993001344 00000100:00100000:3.0:1783520931.019742:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43884 00000100:00100000:3.0:1783520931.019755:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94360550c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+838:3628:x1870152993001344:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.019770:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2142 count 1 00010000:00010000:3.0:1783520931.019785:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc0247ae lrc: 3/0,1 mode: --/PW res: [0x280000401:0x85e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.019801:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 000000005dfd9b84/0xd5401cbabc021d4c lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x85e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2531 expref: 194 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.019827:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x85e:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.019836:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x85e:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.019843:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc0247ae lrc: 4/0,1 mode: --/PW res: [0x280000401:0x85e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.019871:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 000000005dfd9b84/0xd5401cbabc021d4c lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x85e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2531 expref: 195 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.019884:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 000000005dfd9b84/0xd5401cbabc021d4c lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x85e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2531 expref: 196 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.019908:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515301c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0000_UUID:35282:1870152993001856:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.019938:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af1000 going to sleep for 11 seconds 00000100:00100000:3.0:1783520931.019956:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002322304 00000100:00100000:3.0:1783520931.019961:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002322304 00010000:00010000:3.0:1783520931.019979:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 0000000096629104/0xd5401cbabc021ca4 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x85c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2515 expref: 198 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.020014:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35478 00000100:00100000:3.0:1783520931.020029:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507938e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+198:2388:x1870153002322048:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.020043:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.020049:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000d9f98189/0xd5401cbabc021b85 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x859:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24eb expref: 198 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.020086:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000d9f98189/0xd5401cbabc021b85 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x859:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24eb expref: 197 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.020112:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000d9f98189/0xd5401cbabc021b85 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x859:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24eb expref: 197 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.020141:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x859:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.020146:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x859:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.020153:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc024784 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x859:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.020162:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc024784 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x859:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.020181:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.020204:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507938e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+194:2388:x1870153002322048:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 177us (452us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.020224:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35478 00010000:00010000:3.0:1783520931.020492:0:20415:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000096629104/0xd5401cbabc021ca4 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x85c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2515 expref: 191 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.020555:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c552680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0000_UUID:20415:1870152993001088:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.020576:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000003a22489f/0xd5401cbabc024799 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x85c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.020611:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.020619:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000003a22489f/0xd5401cbabc024799 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x85c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.020633:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000003a22489f/0xd5401cbabc024799 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x85c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.020656:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000003a22489f/0xd5401cbabc024799 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x85c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.020678:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000003a22489f/0xd5401cbabc024799 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x85c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.020699:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520931.020710:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.020717:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000003a22489f/0xd5401cbabc024799 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x85c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.020726:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.020732:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000003a22489f/0xd5401cbabc024799 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x85c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.020748:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000003a22489f/0xd5401cbabc024799 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x85c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.020766:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000003a22489f/0xd5401cbabc024799 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x85c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.020832:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000003a22489f/0xd5401cbabc024799 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x85c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.020854:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.020858:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.020862:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.020936:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c550a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+842:3626:x1870152993000832:12345-0@lo:6:osp-syn-0-0.0 Request processed in 2760us (2826us total) trans 4294977364 rc 0/0 00000100:00100000:3.0:1783520931.020946:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43881 00000100:00100000:3.0:1783520931.021032:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993002496 00000100:00100000:3.0:1783520931.021038:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993002496 00000100:00100000:3.0:1783520931.021064:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43888 00000100:00100000:3.0:1783520931.021075:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516682680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+842:3628:x1870152993002496:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.021087:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2147 count 1 00010000:00010000:3.0:1783520931.021099:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc0247ca lrc: 3/0,1 mode: --/PW res: [0x280000401:0x863:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.021112:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 000000007604ef44/0xd5401cbabc021e5d lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x863:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2577 expref: 190 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.021120:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x863:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.021125:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x863:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.021132:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc0247ca lrc: 4/0,1 mode: --/PW res: [0x280000401:0x863:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.021156:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 000000007604ef44/0xd5401cbabc021e5d lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x863:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2577 expref: 190 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.021223:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 000000007604ef44/0xd5401cbabc021e5d lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x863:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2577 expref: 190 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.021272:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515303480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0000_UUID:20415:1870152993003008:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.021350:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af13c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520931.021684:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515301c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0000_UUID:35282:1870152993001856:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.021698:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc0247ae lrc: 3/0,1 mode: --/PW res: [0x280000401:0x85e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.021855:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002322688 00000100:00100000:3.0:1783520931.021859:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002322688 00010000:00010000:3.0:1783520931.021878:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000000ecdb44a/0xd5401cbabc021dc3 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x85f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f253f expref: 191 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.021901:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002322560 00000100:00100000:3.0:1783520931.021904:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002322560 00010000:00010000:3.0:1783520931.021915:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000005dfd9b84/0xd5401cbabc021d4c lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x85e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2531 expref: 192 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.021936:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35482 00000100:00100000:3.0:1783520931.021995:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c550700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+190:2387:x1870153002322560:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.022004:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.022008:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000005dfd9b84/0xd5401cbabc021d4c lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x85e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2531 expref: 190 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.022018:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000005dfd9b84/0xd5401cbabc021d4c lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x85e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2531 expref: 190 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.022032:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000005dfd9b84/0xd5401cbabc021d4c lrc: 0/0,0 mode: --/PR res: [0x280000401:0x85e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2531 expref: 190 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.022090:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x85e:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.022094:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x85e:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.022098:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc0247ae lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x85e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.022109:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc0247ae lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x85e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.022132:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.022157:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c550700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+188:2387:x1870153002322560:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 212us (284us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.022208:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35482 00000100:00100000:3.0:1783520931.022229:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002322816 00000100:00100000:3.0:1783520931.022235:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002322816 00010000:00010000:3.0:1783520931.022245:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.022259:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc0247ae lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x85e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.022284:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc0247ae lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x85e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.022299:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc0247ae lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x85e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.022321:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc0247ae lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x85e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.022341:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to regular blp list, 0 blwis in pool 00000100:00100000:3.0:1783520931.022617:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94360550c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+846:3628:x1870152993001344:12345-0@lo:6:osp-syn-0-0.0 Request processed in 2864us (3269us total) trans 4294977367 rc 0/0 00000100:00100000:3.0:1783520931.022629:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43884 00010000:00010000:3.0:1783520931.022665:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000c6bcb9a2/0xd5401cbabc021e02 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x860:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f254d expref: 188 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.022692:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35483 00000100:00100000:3.0:1783520931.022701:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638585850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+188:2387:x1870153002322816:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.022716:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.022723:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000c6bcb9a2/0xd5401cbabc021e02 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x860:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f254d expref: 188 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.022789:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000c6bcb9a2/0xd5401cbabc021e02 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x860:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f254d expref: 189 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.022814:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c6bcb9a2/0xd5401cbabc021e02 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x860:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f254d expref: 189 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.022829:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x860:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.022837:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x860:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.022846:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000c8b052a2/0xd5401cbabc0247b5 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x860:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.022854:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000c8b052a2/0xd5401cbabc0247b5 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x860:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.022873:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35484 00000100:00100000:3.0:1783520931.022886:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507939c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+191:2390:x1870153002323200:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.022899:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.022906:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000a20cf329/0xd5401cbabc021e6b lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x864:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2585 expref: 191 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.022921:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000a20cf329/0xd5401cbabc021e6b lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x864:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2585 expref: 191 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.022944:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x864:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.022952:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x864:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.022962:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 0000000018903c53/0xd5401cbabc0247d1 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x864:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.022994:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000018903c53/0xd5401cbabc0247d1 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x864:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.023024:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.023099:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507939c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+190:2390:x1870153002323200:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 167us (323us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.023116:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35484 00000100:00100000:3.0:1783520931.023136:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35487 00000100:00100000:3.0:1783520931.023144:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507939880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+189:2387:x1870153002323072:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.023152:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.023160:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 0000000007c3da39/0xd5401cbabc021e3a lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x861:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f255b expref: 188 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.023215:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 0000000007c3da39/0xd5401cbabc021e3a lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x861:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f255b expref: 188 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.023249:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x861:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.023258:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x861:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.023269:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000ddf44b9d/0xd5401cbabc0247c3 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x861:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.023326:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000ddf44b9d/0xd5401cbabc0247c3 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x861:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.023348:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520931.023352:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc0247bc lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x862:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.023420:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.023425:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc0247bc lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x862:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.023434:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc0247bc lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x862:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.023448:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc0247bc lrc: 1/0,0 mode: --/PW res: [0x280000401:0x862:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.023459:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc0247bc lrc: 0/0,0 mode: --/PW res: [0x280000401:0x862:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.023485:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.023490:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 0000000018903c53/0xd5401cbabc0247d1 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x864:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.023536:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.023543:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 0000000018903c53/0xd5401cbabc0247d1 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x864:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.023574:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 0000000018903c53/0xd5401cbabc0247d1 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x864:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.023607:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 0000000018903c53/0xd5401cbabc0247d1 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x864:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.023623:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000018903c53/0xd5401cbabc0247d1 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x864:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.023645:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.023649:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.023652:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.023659:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.023670:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.023701:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507939880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+186:2387:x1870153002323072:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 557us (854us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.023713:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35487 00010000:00010000:3.0:1783520931.023754:0:20415:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000007604ef44/0xd5401cbabc021e5d lrc: 0/0,0 mode: --/PR res: [0x280000401:0x863:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2577 expref: 184 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.023780:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515303480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0000_UUID:20415:1870152993003008:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.023801:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc0247ca lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x863:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.023830:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc0247ca lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x863:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.023942:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc0247ca lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x863:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.023956:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc0247ca lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x863:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.023972:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc0247ca lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x863:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.024050:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520931.024062:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.024071:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993004288 00000100:00100000:3.0:1783520931.024076:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993004288 00000100:00100000:3.0:1783520931.024097:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43894 00000100:00100000:3.0:1783520931.024112:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435130f6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+848:3625:x1870152993004288:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.024131:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2153 count 1 00010000:00010000:3.0:1783520931.024148:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc0247f4 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x869:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.024174:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 000000005f6ce6e6/0xd5401cbabc0221f2 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x869:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f25cb expref: 182 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.024191:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x869:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.024232:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x869:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.024238:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc0247f4 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x869:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.024268:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 000000005f6ce6e6/0xd5401cbabc0221f2 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x869:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25cb expref: 183 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.024336:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 000000005f6ce6e6/0xd5401cbabc0221f2 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x869:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25cb expref: 183 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.024354:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515303480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0000_UUID:35282:1870152993004544:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.024434:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af10c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520931.024568:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516682680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+850:3628:x1870152993002496:12345-0@lo:6:osp-syn-0-0.0 Request processed in 3493us (3573us total) trans 4294977371 rc 0/0 00000100:00100000:3.0:1783520931.024576:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43888 00010000:00010000:3.0:1783520931.024619:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.024644:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638585850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+181:2387:x1870153002322816:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 1943us (2549us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.024652:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35483 00000100:00100000:3.0:1783520931.024754:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993004800 00000100:00100000:3.0:1783520931.024760:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993004800 00010000:00010000:3.0:1783520931.024772:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.024780:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000c8b052a2/0xd5401cbabc0247b5 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x860:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.024792:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.024799:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000c8b052a2/0xd5401cbabc0247b5 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x860:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.024829:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000c8b052a2/0xd5401cbabc0247b5 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x860:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.024886:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000c8b052a2/0xd5401cbabc0247b5 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x860:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.024904:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c8b052a2/0xd5401cbabc0247b5 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x860:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.024927:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.024929:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.024932:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.025001:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43896 00010000:00010000:3.0:1783520931.025011:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.025019:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000841b1b91/0xd5401cbabc0247e6 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x867:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.025031:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.025037:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000841b1b91/0xd5401cbabc0247e6 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x867:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.025049:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000841b1b91/0xd5401cbabc0247e6 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x867:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.025071:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000841b1b91/0xd5401cbabc0247e6 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x867:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.025080:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000841b1b91/0xd5401cbabc0247e6 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x867:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.025094:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.025098:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.025101:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.025111:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507938380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+852:3627:x1870152993004800:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.025125:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2155 count 1 00010000:00010000:3.0:1783520931.025136:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024809 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x86b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.025147:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 0000000023a63ee8/0xd5401cbabc021ec6 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x86b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f25e7 expref: 179 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.025154:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x86b:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.025157:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x86b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.025162:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024809 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x86b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.025179:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 0000000023a63ee8/0xd5401cbabc021ec6 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x86b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25e7 expref: 179 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.025190:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 0000000023a63ee8/0xd5401cbabc021ec6 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x86b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25e7 expref: 179 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.025296:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515301c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0000_UUID:8408:1870152993005184:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.025326:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af1000 going to sleep for 11 seconds 00000100:00100000:3.0:1783520931.025402:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43899 00000100:00100000:3.0:1783520931.025414:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515f0a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+853:3628:x1870152993005440:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.025428:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2158 count 1 00010000:00010000:3.0:1783520931.025445:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc024817 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x86e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.025564:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000e963a7af/0xd5401cbabc021ef7 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x86e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2611 expref: 179 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.025572:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x86e:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.025576:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x86e:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.025580:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc024817 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x86e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.025616:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000e963a7af/0xd5401cbabc021ef7 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x86e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2611 expref: 179 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.025628:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000e963a7af/0xd5401cbabc021ef7 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x86e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2611 expref: 179 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.025640:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515302300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0000_UUID:20415:1870152993005696:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.025672:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af1600 going to sleep for 11 seconds 00000100:00100000:3.0:1783520931.025894:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515303480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0000_UUID:35282:1870152993004544:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.025952:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc0247f4 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x869:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.025990:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002324096 00000100:00100000:3.0:1783520931.025993:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002324096 00010000:00010000:3.0:1783520931.026057:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000fb2a4d36/0xd5401cbabc021eb8 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x86a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25d9 expref: 182 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.026128:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35493 00000100:00100000:3.0:1783520931.026140:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e47b800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+182:2390:x1870153002324096:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.026151:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.026160:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000fb2a4d36/0xd5401cbabc021eb8 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x86a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25d9 expref: 182 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.026187:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000fb2a4d36/0xd5401cbabc021eb8 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x86a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25d9 expref: 181 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.026216:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000fb2a4d36/0xd5401cbabc021eb8 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x86a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25d9 expref: 181 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.026229:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x86a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.026234:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x86a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.026238:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000c721a958/0xd5401cbabc0247fb lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x86a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.026248:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000c721a958/0xd5401cbabc0247fb lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x86a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.026278:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.026353:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e47b800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+177:2390:x1870153002324096:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 212us (378us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.026380:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35493 00000100:00100000:3.0:1783520931.026475:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515301c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0000_UUID:8408:1870152993005184:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.026490:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024809 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x86b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.026541:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.026547:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc0247f4 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x869:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.026557:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784de00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.026566:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc0247f4 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x869:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.026575:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc0247f4 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x869:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.026707:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc0247f4 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x869:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.026729:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc0247f4 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x869:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.026746:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.026750:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.026754:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.026774:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35496 00000100:00100000:3.0:1783520931.026788:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507939500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+179:2387:x1870153002324480:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.026802:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.026810:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 0000000072faac44/0xd5401cbabc021ee9 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x86d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2603 expref: 179 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.026826:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 0000000072faac44/0xd5401cbabc021ee9 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x86d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2603 expref: 179 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.026851:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000072faac44/0xd5401cbabc021ee9 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x86d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2603 expref: 178 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.026876:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x86d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.026883:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x86d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.026888:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024810 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x86d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.026911:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024810 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x86d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.026929:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.026954:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507939500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+177:2387:x1870153002324480:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 168us (363us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.027019:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35496 00010000:00010000:3.0:1783520931.027044:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024810 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x86d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.027073:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024810 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x86d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.027097:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024810 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x86d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.027105:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024810 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x86d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.027118:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520931.027131:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.027189:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024810 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x86d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.027201:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.027205:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024810 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x86d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.027255:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024810 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x86d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.027269:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024810 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x86d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.027281:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024810 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x86d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.027299:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.027333:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.027335:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.027493:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507938000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+857:3626:x1870152993005312:12345-0@lo:6:osp-syn-0-0.0 Request processed in 2098us (2204us total) trans 4294977378 rc 0/0 00000100:00100000:3.0:1783520931.027501:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43898 00000004:00080000:3.0:1783520931.027527:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515f09500/2, rc 0, transno 4294977378 00000100:00100000:3.0:1783520931.027536:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515f09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993005312:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.027630:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515301c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993006592:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.027657:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520931.027700:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024809 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x86b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.027710:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024809 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x86b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.027734:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024809 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x86b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.027742:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024809 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x86b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.027783:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1783520931.027792:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515302d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993006720:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.027820:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520931.027855:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520931.027969:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.028041:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507938380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+859:3627:x1870152993004800:12345-0@lo:6:osp-syn-0-0.0 Request processed in 3031us (3339us total) trans 4294977381 rc 0/0 00000100:00100000:3.0:1783520931.028050:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43896 00010000:00010000:3.0:1783520931.028104:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.028113:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000a5d93cec/0xd5401cbabc024802 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x86c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.028182:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.028189:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000a5d93cec/0xd5401cbabc024802 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x86c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.028198:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000a5d93cec/0xd5401cbabc024802 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x86c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.028233:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000a5d93cec/0xd5401cbabc024802 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x86c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.028290:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000a5d93cec/0xd5401cbabc024802 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x86c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.028311:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.028315:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.028318:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520931.028391:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e46f800/2, rc 0, transno 4294977381 00000100:00100000:3.0:1783520931.028438:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e46f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993004800:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.028489:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515300000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993007488:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.028524:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.028577:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43907 00000100:00100000:3.0:1783520931.028601:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515300700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+861:3627:x1870152993007488:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.028724:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2166 count 1 00010000:00010000:3.0:1783520931.028738:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc02484f lrc: 3/0,1 mode: --/PW res: [0x280000401:0x876:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.028758:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000f49f4a15/0xd5401cbabc021f83 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x876:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2681 expref: 171 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.028769:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x876:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.028774:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x876:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.028779:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc02484f lrc: 4/0,1 mode: --/PW res: [0x280000401:0x876:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.028807:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000f49f4a15/0xd5401cbabc021f83 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x876:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2681 expref: 171 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.028829:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000f49f4a15/0xd5401cbabc021f83 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x876:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2681 expref: 171 pid: 35282 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.028853:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515300e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0000_UUID:35274:1870152993007744:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.028957:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af10c0 going to sleep for 11 seconds 00000004:00080000:3.0:1783520931.028987:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515f09c00/2, rc 0, transno 4294977383 00000100:00100000:3.0:1783520931.029049:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515f09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993006208:0@lo:6:osp-syn-0-0.0 00010000:00010000:3.0:1783520931.029059:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.029066:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000001e04e98c/0xd5401cbabc024825 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x870:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.029157:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.029162:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000001e04e98c/0xd5401cbabc024825 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x870:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.029171:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000001e04e98c/0xd5401cbabc024825 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x870:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.029192:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000001e04e98c/0xd5401cbabc024825 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x870:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.029211:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000001e04e98c/0xd5401cbabc024825 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x870:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.029226:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.029230:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.029232:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520931.029291:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515f08e00/2, rc 0, transno 4294977384 00000100:00100000:3.0:1783520931.029303:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515f08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993006080:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.029527:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993008000 00000100:00100000:3.0:1783520931.029636:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993008000 00010000:00010000:3.0:1783520931.029687:0:20415:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000e963a7af/0xd5401cbabc021ef7 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x86e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2611 expref: 169 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.029707:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515302300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0000_UUID:20415:1870152993005696:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.029726:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc024817 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x86e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.029796:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc024817 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x86e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.029819:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc024817 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x86e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.029890:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc024817 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x86e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.029899:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc024817 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x86e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.029952:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520931.029960:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.029966:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc024817 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x86e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.029986:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784d000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.029990:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc024817 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x86e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.029999:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc024817 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x86e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.030009:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc024817 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x86e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.030017:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc024817 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x86e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.030085:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.030088:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.030090:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.030167:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515f0a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+866:3628:x1870152993005440:12345-0@lo:6:osp-syn-0-0.0 Request processed in 4754us (4844us total) trans 4294977386 rc 0/0 00000100:00100000:3.0:1783520931.030179:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43899 00000100:00100000:3.0:1783520931.030329:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515300e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0000_UUID:35274:1870152993007744:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.030344:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc02484f lrc: 3/0,1 mode: --/PW res: [0x280000401:0x876:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.030473:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.030530:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc024841 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x874:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.030539:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.030544:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc024841 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x874:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.030553:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc024841 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x874:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.030569:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc024841 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x874:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.030577:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc024841 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x874:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.030610:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.030614:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.030617:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.030719:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993008640 00000100:00100000:3.0:1783520931.030725:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993008640 00000100:00100000:3.0:1783520931.030752:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993008768 00000100:00100000:3.0:1783520931.030757:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993008768 00000100:00100000:3.0:1783520931.030776:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993008896 00000100:00100000:3.0:1783520931.030778:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993008896 00000100:00100000:3.0:1783520931.030804:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43911 00000100:00100000:3.0:1783520931.030882:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515f09880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+867:3628:x1870152993008640:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.030902:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2170 count 1 00010000:00010000:3.0:1783520931.030915:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc02486b lrc: 3/0,1 mode: --/PW res: [0x280000401:0x87a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.030927:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000472990b3/0xd5401cbabc021f98 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x87a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f26b9 expref: 165 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.030947:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x87a:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.030953:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x87a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.030962:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc02486b lrc: 4/0,1 mode: --/PW res: [0x280000401:0x87a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.030996:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000472990b3/0xd5401cbabc021f98 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x87a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26b9 expref: 165 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.031025:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000472990b3/0xd5401cbabc021f98 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x87a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26b9 expref: 165 pid: 35282 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.031101:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515302680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0000_UUID:8408:1870152993009024:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.031131:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af10c0 going to sleep for 11 seconds 00010000:00010000:3.0:1783520931.031153:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc02484f lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x876:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.031166:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc02484f lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x876:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.031182:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc02484f lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x876:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.031191:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc02484f lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x876:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.031250:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520931.031263:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.031270:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc02484f lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x876:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.031285:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784ce00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.031291:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc02484f lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x876:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.031344:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc02484f lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x876:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.031377:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc02484f lrc: 1/0,0 mode: --/PW res: [0x280000401:0x876:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.031414:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc02484f lrc: 0/0,0 mode: --/PW res: [0x280000401:0x876:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.031445:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.031450:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc024856 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x877:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.031474:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.031482:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc024856 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x877:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.031506:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc024856 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x877:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.031532:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc024856 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x877:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.031548:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc024856 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x877:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.031579:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.031584:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024864 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x878:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.031605:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.031609:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024864 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x878:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.031619:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024864 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x878:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.031631:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024864 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x878:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.031639:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024864 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x878:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.031659:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.031663:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.031668:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.031676:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.031822:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515300700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+871:3627:x1870152993007488:12345-0@lo:6:osp-syn-0-0.0 Request processed in 3236us (3300us total) trans 4294977392 rc 0/0 00000100:00100000:3.0:1783520931.031829:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43907 00000004:00080000:3.0:1783520931.031851:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515300000/2, rc 0, transno 4294977392 00000100:00100000:3.0:1783520931.031865:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515300000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993007488:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.031986:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993009920 00000100:00100000:3.0:1783520931.031990:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993009920 00000100:00100000:3.0:1783520931.032015:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43917 00000100:00100000:3.0:1783520931.032026:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515304380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+871:3625:x1870152993009920:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.032037:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2176 count 1 00010000:00010000:3.0:1783520931.032048:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024895 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x880:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.032059:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000c2388984/0xd5401cbabc022001 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x880:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f270d expref: 162 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.032066:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x880:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.032069:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x880:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.032073:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024895 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x880:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.032091:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000c2388984/0xd5401cbabc022001 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x880:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f270d expref: 162 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.032101:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000c2388984/0xd5401cbabc022001 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x880:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f270d expref: 162 pid: 12734 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.032150:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c555500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0000_UUID:35274:1870152993010304:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.032179:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af1600 going to sleep for 11 seconds 00000100:00100000:3.0:1783520931.032447:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515f08380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0000_UUID:35275:1870152993008384:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.032524:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000007b280ca1/0xd5401cbabc02485d lrc: 3/0,1 mode: --/PW res: [0x280000401:0x879:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.032549:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515302680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0000_UUID:8408:1870152993009024:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.032560:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc02486b lrc: 3/0,1 mode: --/PW res: [0x280000401:0x87a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.032603:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002326016 00000100:00100000:3.0:1783520931.032607:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002326016 00010000:00010000:3.0:1783520931.032692:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000c865fb8d/0xd5401cbabc021f9f lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x879:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26ab expref: 164 pid: 8409 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.032715:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002326400 00000100:00100000:3.0:1783520931.032716:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002326400 00010000:00010000:3.0:1783520931.032728:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000472990b3/0xd5401cbabc021f98 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x87a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26b9 expref: 166 pid: 35282 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.032761:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35510 00000100:00100000:3.0:1783520931.032770:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943629897c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+166:2388:x1870153002326400:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.032780:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.032786:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000472990b3/0xd5401cbabc021f98 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x87a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26b9 expref: 166 pid: 35282 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.032813:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000472990b3/0xd5401cbabc021f98 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x87a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26b9 expref: 166 pid: 35282 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.032843:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000472990b3/0xd5401cbabc021f98 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x87a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26b9 expref: 166 pid: 35282 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.032868:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x87a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.032872:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x87a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.032879:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc02486b lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x87a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.032899:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc02486b lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x87a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.032926:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.032951:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943629897c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+163:2388:x1870153002326400:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 181us (286us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.032968:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35510 00010000:00010000:3.0:1783520931.032999:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000007b280ca1/0xd5401cbabc02485d lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x879:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.033016:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000007b280ca1/0xd5401cbabc02485d lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x879:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.033037:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000007b280ca1/0xd5401cbabc02485d lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x879:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.033058:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000007b280ca1/0xd5401cbabc02485d lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x879:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.033083:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520931.033161:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515f08700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+872:3625:x1870152993008128:12345-0@lo:6:osp-syn-0-0.0 Request processed in 3506us (3593us total) trans 4294977393 rc 0/0 00000100:00100000:3.0:1783520931.033170:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43909 00010000:00010000:3.0:1783520931.033250:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc02486b lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x87a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.033269:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc02486b lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x87a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.033278:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc02486b lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x87a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.033296:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc02486b lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x87a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.033317:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520931.033326:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.033332:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc02486b lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x87a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.033338:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784c200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.033342:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc02486b lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x87a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.033351:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc02486b lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x87a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.033386:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc02486b lrc: 1/0,0 mode: --/PW res: [0x280000401:0x87a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.033398:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc02486b lrc: 0/0,0 mode: --/PW res: [0x280000401:0x87a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.033420:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.033423:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.033425:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.033446:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c555500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0000_UUID:35274:1870152993010304:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.033459:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024895 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x880:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.033609:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515f09880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+872:3628:x1870152993008640:12345-0@lo:6:osp-syn-0-0.0 Request processed in 2796us (3117us total) trans 4294977395 rc 0/0 00000100:00100000:3.0:1783520931.033626:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43911 00000100:00100000:3.0:1783520931.033644:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993010432 00000100:00100000:3.0:1783520931.033647:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993010432 00000100:00100000:3.0:1783520931.033679:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993010560 00000100:00100000:3.0:1783520931.033683:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993010560 00000100:00100000:3.0:1783520931.033710:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43918 00000100:00100000:3.0:1783520931.033723:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516682680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+873:3626:x1870152993010432:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.033738:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2177 count 1 00010000:00010000:3.0:1783520931.033752:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc02489c lrc: 3/0,1 mode: --/PW res: [0x280000401:0x881:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.033783:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 000000003237d771/0xd5401cbabc022008 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x881:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f271b expref: 161 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.033806:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x881:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.033810:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x881:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.033817:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc02489c lrc: 4/0,1 mode: --/PW res: [0x280000401:0x881:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.033840:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.033845:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000c42e258a/0xd5401cbabc024872 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x87b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.033860:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.033866:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000c42e258a/0xd5401cbabc024872 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x87b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.033891:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000c42e258a/0xd5401cbabc024872 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x87b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.033913:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000c42e258a/0xd5401cbabc024872 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x87b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.033923:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c42e258a/0xd5401cbabc024872 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x87b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.033941:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.033943:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.033947:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.033974:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35514 00000100:00100000:3.0:1783520931.033988:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507939180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+158:2390:x1870153002326912:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.033999:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.034005:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000006b5f5bf2/0xd5401cbabc021ff3 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x87f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26ff expref: 158 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.034023:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000006b5f5bf2/0xd5401cbabc021ff3 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x87f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26ff expref: 158 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.034037:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000006b5f5bf2/0xd5401cbabc021ff3 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x87f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26ff expref: 158 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.034056:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x87f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.034064:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x87f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.034071:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc02488e lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x87f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.034096:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc02488e lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x87f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.034127:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.034194:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507939180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+157:2390:x1870153002326912:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 210us (613us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.034213:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35514 00000100:00100000:3.0:1783520931.034228:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002326656 00000100:00100000:3.0:1783520931.034234:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002326656 00010000:00010000:3.0:1783520931.034252:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000c2388984/0xd5401cbabc022001 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x880:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f270d expref: 157 pid: 12734 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.034286:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35515 00000100:00100000:3.0:1783520931.034299:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350793a680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+157:2387:x1870153002326656:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.034314:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.034323:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000c2388984/0xd5401cbabc022001 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x880:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f270d expref: 156 pid: 12734 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.034425:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000c2388984/0xd5401cbabc022001 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x880:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f270d expref: 156 pid: 12734 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.034458:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c2388984/0xd5401cbabc022001 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x880:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f270d expref: 156 pid: 12734 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.034486:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x880:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.034495:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x880:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.034506:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024895 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x880:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.034525:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024895 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x880:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.034554:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.034567:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.034603:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350793a680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+155:2387:x1870153002326656:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 293us (564us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.034613:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35515 00010000:00010000:3.0:1783520931.034638:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024895 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x880:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.034657:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024895 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x880:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.034669:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024895 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x880:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.034683:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024895 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x880:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.034698:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520931.034706:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.034712:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024895 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x880:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.034722:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784ca00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.034726:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024895 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x880:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.034734:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024895 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x880:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.034748:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024895 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x880:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.034761:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024895 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x880:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.034787:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.034791:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.034796:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.034886:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515304380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+879:3625:x1870152993009920:12345-0@lo:6:osp-syn-0-0.0 Request processed in 2859us (2939us total) trans 4294977400 rc 0/0 00000100:00100000:3.0:1783520931.034909:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43917 00010000:00010000:3.0:1783520931.034954:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 000000003237d771/0xd5401cbabc022008 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x881:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f271b expref: 154 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.034980:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 000000003237d771/0xd5401cbabc022008 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x881:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f271b expref: 154 pid: 35274 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.035005:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c555500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0000_UUID:8408:1870152993011840:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.035042:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af1600 going to sleep for 11 seconds 00000100:00100000:3.0:1783520931.035319:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993012224 00000100:00100000:3.0:1783520931.035399:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993012224 00000100:00100000:3.0:1783520931.037023:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c555500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0000_UUID:8408:1870152993011840:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.037112:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc02489c lrc: 3/0,1 mode: --/PW res: [0x280000401:0x881:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.037217:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002327040 00000100:00100000:3.0:1783520931.037224:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002327040 00010000:00010000:3.0:1783520931.037252:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000f760c3f7/0xd5401cbabc02201d lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x882:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2729 expref: 159 pid: 35274 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.037310:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002327552 00000100:00100000:3.0:1783520931.037317:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002327552 00010000:00010000:3.0:1783520931.037338:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000001b0cc565/0xd5401cbabc022055 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x886:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2761 expref: 161 pid: 35274 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.037386:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35517 00000100:00100000:3.0:1783520931.037444:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507938a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+161:2387:x1870153002327424:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.037459:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.037476:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 0000000084e3dcad/0xd5401cbabc022040 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x885:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2753 expref: 160 pid: 35274 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.037501:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 0000000084e3dcad/0xd5401cbabc022040 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x885:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2753 expref: 161 pid: 35274 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.037529:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000084e3dcad/0xd5401cbabc022040 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x885:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2753 expref: 161 pid: 35274 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.037560:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x885:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.037576:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x885:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.037605:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc0248b8 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x885:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.037648:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc0248b8 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x885:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.037766:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520931.037783:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.037791:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000be742370/0xd5401cbabc0248aa lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x883:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.037850:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.037861:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000be742370/0xd5401cbabc0248aa lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x883:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.037894:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000be742370/0xd5401cbabc0248aa lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x883:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.037930:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000be742370/0xd5401cbabc0248aa lrc: 1/0,0 mode: --/PW res: [0x280000401:0x883:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.037942:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000be742370/0xd5401cbabc0248aa lrc: 0/0,0 mode: --/PW res: [0x280000401:0x883:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.037964:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.037969:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.037973:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.038008:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507938a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+157:2387:x1870153002327424:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 610us (950us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.038039:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35517 00000100:00100000:3.0:1783520931.038068:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35522 00000100:00100000:3.0:1783520931.038093:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350626e300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+155:2387:x1870153002327552:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.038116:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.038133:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000001b0cc565/0xd5401cbabc022055 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x886:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2761 expref: 155 pid: 35274 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.038165:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000001b0cc565/0xd5401cbabc022055 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x886:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2761 expref: 155 pid: 35274 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.038208:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x886:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.038215:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x886:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.038224:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000c721a958/0xd5401cbabc0248bf lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x886:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.038268:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000c721a958/0xd5401cbabc0248bf lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x886:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.038317:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.038344:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350626e300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+153:2387:x1870153002327552:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 258us (1072us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.038392:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35522 00000100:00100000:3.0:1783520931.038420:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993012480 00000100:00100000:3.0:1783520931.038429:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993012480 00000100:00100000:3.0:1783520931.038465:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43926 00000100:00100000:3.0:1783520931.038481:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e451f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+880:3626:x1870152993012480:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.038506:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2185 count 1 00010000:00010000:3.0:1783520931.038528:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0248d4 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x889:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.038580:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 000000000d8d5e36/0xd5401cbabc0220e8 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x889:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f278b expref: 150 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.038618:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x889:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.038622:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x889:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.038627:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0248d4 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x889:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.038669:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 000000000d8d5e36/0xd5401cbabc0220e8 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x889:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f278b expref: 148 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.038703:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 000000000d8d5e36/0xd5401cbabc0220e8 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x889:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f278b expref: 148 pid: 35282 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.038719:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c555500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0000_UUID:35274:1870152993012608:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.038791:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af1600 going to sleep for 11 seconds 00010000:00010000:3.0:1783520931.038835:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc0248b8 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x885:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.038846:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc0248b8 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x885:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.038857:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc0248b8 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x885:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.038870:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc0248b8 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x885:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.038887:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520931.038898:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.038905:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc0248b8 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x885:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.038914:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.038921:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc0248b8 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x885:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.038947:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc0248b8 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x885:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.038967:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc0248b8 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x885:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.038981:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc0248b8 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x885:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.039094:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.039100:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000d4a09c64/0xd5401cbabc0248cd lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x888:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.039128:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.039134:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000d4a09c64/0xd5401cbabc0248cd lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x888:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.039144:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000d4a09c64/0xd5401cbabc0248cd lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x888:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.039158:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000d4a09c64/0xd5401cbabc0248cd lrc: 1/0,0 mode: --/PW res: [0x280000401:0x888:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.039191:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000d4a09c64/0xd5401cbabc0248cd lrc: 0/0,0 mode: --/PW res: [0x280000401:0x888:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.039230:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.039237:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.039240:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.039249:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.039345:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c56a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+883:3626:x1870152993011328:12345-0@lo:6:osp-syn-0-0.0 Request processed in 4500us (4627us total) trans 4294977405 rc 0/0 00000100:00100000:3.0:1783520931.039403:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43922 00000100:00100000:3.0:1783520931.039421:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993012992 00000100:00100000:3.0:1783520931.039427:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993012992 00000100:00100000:3.0:1783520931.039457:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43928 00000100:00100000:3.0:1783520931.039470:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e450700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+884:3625:x1870152993012992:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.039492:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2188 count 1 00010000:00010000:3.0:1783520931.039512:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc0248e2 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x88c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.039547:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000befd27b9/0xd5401cbabc0220f6 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x88c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f27b5 expref: 147 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.039571:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x88c:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.039578:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x88c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.039584:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc0248e2 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x88c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.039792:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000befd27b9/0xd5401cbabc0220f6 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x88c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27b5 expref: 146 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.039809:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000befd27b9/0xd5401cbabc0220f6 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x88c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27b5 expref: 146 pid: 12734 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.039826:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c554e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0000_UUID:35276:1870152993013504:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.039930:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af10c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520931.040089:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c555500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0000_UUID:35274:1870152993012608:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.040118:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0248d4 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x889:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.040189:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002328064 00000100:00100000:3.0:1783520931.040198:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002328064 00010000:00010000:3.0:1783520931.040223:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000000d8d5e36/0xd5401cbabc0220e8 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x889:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f278b expref: 147 pid: 35282 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.040263:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35524 00000100:00100000:3.0:1783520931.040274:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e470e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+147:2387:x1870153002328064:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.040283:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.040288:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000000d8d5e36/0xd5401cbabc0220e8 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x889:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f278b expref: 147 pid: 35282 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.040308:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000000d8d5e36/0xd5401cbabc0220e8 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x889:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f278b expref: 147 pid: 35282 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.040475:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000000d8d5e36/0xd5401cbabc0220e8 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x889:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f278b expref: 147 pid: 35282 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.040505:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x889:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.040621:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x889:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.040632:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0248d4 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x889:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.040645:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0248d4 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x889:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.040742:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.040776:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e470e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+146:2387:x1870153002328064:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 501us (781us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.040793:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35524 00000100:00100000:3.0:1783520931.040825:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002328192 00000100:00100000:3.0:1783520931.040831:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002328192 00010000:00010000:3.0:1783520931.040931:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000003c512cf6/0xd5401cbabc0220ef lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x88b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27a7 expref: 146 pid: 35274 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.041175:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0248d4 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x889:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.041195:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0248d4 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x889:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.041218:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0248d4 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x889:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.041296:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0248d4 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x889:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.041326:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520931.041342:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.041347:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0248d4 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x889:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.041356:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784ce00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.041389:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0248d4 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x889:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.041397:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0248d4 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x889:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.041413:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0248d4 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x889:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.041420:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc0248d4 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x889:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.041458:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.041463:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.041466:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.041627:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e451f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+887:3626:x1870152993012480:12345-0@lo:6:osp-syn-0-0.0 Request processed in 3149us (3525us total) trans 4294977408 rc 0/0 00000100:00100000:3.0:1783520931.041636:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43926 00000004:00080000:3.0:1783520931.041677:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e450e00/2, rc 0, transno 4294977408 00000100:00100000:3.0:1783520931.041693:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e450e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993012480:0@lo:6:osp-syn-0-0.0 00010000:00010000:3.0:1783520931.041812:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.041820:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc0248db lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x88b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.041830:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.041836:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc0248db lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x88b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.041852:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc0248db lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x88b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.041885:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc0248db lrc: 1/0,0 mode: --/PW res: [0x280000401:0x88b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.041896:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc0248db lrc: 0/0,0 mode: --/PW res: [0x280000401:0x88b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.041989:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.041994:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.041997:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.042113:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c554e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993014400:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.042160:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.042203:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993014400 00000100:00100000:3.0:1783520931.042211:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993014400 00000100:00100000:3.0:1783520931.042254:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43934 00000100:00100000:3.0:1783520931.042265:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515302680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+888:3626:x1870152993014400:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.042281:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2193 count 1 00010000:00010000:3.0:1783520931.042298:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc02490c lrc: 3/0,1 mode: --/PW res: [0x280000401:0x891:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.042315:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 0000000000601eb7/0xd5401cbabc022119 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x891:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f27fb expref: 143 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.042324:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x891:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.042329:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x891:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.042334:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc02490c lrc: 4/0,1 mode: --/PW res: [0x280000401:0x891:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.042418:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 0000000000601eb7/0xd5401cbabc022119 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x891:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27fb expref: 143 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.042430:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 0000000000601eb7/0xd5401cbabc022119 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x891:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27fb expref: 143 pid: 12734 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.042444:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515f08380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0000_UUID:35274:1870152993014784:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.042573:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af1600 going to sleep for 11 seconds 00010000:00010000:3.0:1783520931.042638:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc02489c lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x881:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.042718:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc02489c lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x881:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.042734:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc02489c lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x881:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.042760:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc02489c lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x881:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.042786:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520931.042935:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516682680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+890:3626:x1870152993010432:12345-0@lo:6:osp-syn-0-0.0 Request processed in 9210us (9605us total) trans 4294977411 rc 0/0 00000100:00100000:3.0:1783520931.042944:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43918 00000004:00080000:3.0:1783520931.043137:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94360550c380/2, rc 0, transno 4294977411 00000100:00100000:3.0:1783520931.043156:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94360550c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993010432:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.043301:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35529 00000100:00100000:3.0:1783520931.043407:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507939180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+145:2387:x1870153002328704:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.043427:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.043490:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000006aae121a/0xd5401cbabc0220fd lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x88d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27c3 expref: 143 pid: 35282 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.043522:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000006aae121a/0xd5401cbabc0220fd lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x88d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27c3 expref: 143 pid: 35282 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.043547:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000006aae121a/0xd5401cbabc0220fd lrc: 0/0,0 mode: --/PR res: [0x280000401:0x88d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27c3 expref: 143 pid: 35282 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.043567:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x88d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.043585:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x88d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.043606:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc0248f0 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x88d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.043800:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc0248f0 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x88d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.043825:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520931.043832:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000dc1a697e/0xd5401cbabc0248e9 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x88e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.043847:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.043853:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000dc1a697e/0xd5401cbabc0248e9 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x88e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.043862:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000dc1a697e/0xd5401cbabc0248e9 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x88e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.043906:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000dc1a697e/0xd5401cbabc0248e9 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x88e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.043920:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000dc1a697e/0xd5401cbabc0248e9 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x88e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.043946:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.043952:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc0248fe lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x88f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.043962:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.043969:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc0248fe lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x88f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.043981:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc0248fe lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x88f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.043999:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc0248fe lrc: 1/0,0 mode: --/PW res: [0x280000401:0x88f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.044076:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc0248fe lrc: 0/0,0 mode: --/PW res: [0x280000401:0x88f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.044117:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.044124:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.044130:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.044157:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.044171:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993015296 00000100:00100000:3.0:1783520931.044183:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993015296 00000100:00100000:3.0:1783520931.044340:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43937 00000100:00100000:3.0:1783520931.044398:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ae6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+892:3625:x1870152993015168:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.044449:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2196 count 1 00010000:00010000:3.0:1783520931.044473:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024921 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x894:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.044531:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 0000000081645457/0xd5401cbabc022135 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x894:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2825 expref: 143 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.044562:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x894:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.044567:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x894:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.044577:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024921 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x894:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.044636:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 0000000081645457/0xd5401cbabc022135 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x894:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2825 expref: 142 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.044708:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 0000000081645457/0xd5401cbabc022135 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x894:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2825 expref: 141 pid: 35274 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.044748:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515f08000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0000_UUID:35275:1870152993015552:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.044794:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af1000 going to sleep for 11 seconds 00010000:00010000:3.0:1783520931.044834:0:35274:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000000601eb7/0xd5401cbabc022119 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x891:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27fb expref: 140 pid: 12734 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.044862:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515f08380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0000_UUID:35274:1870152993014784:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.044889:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc02490c lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x891:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.044903:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc02490c lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x891:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.044921:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc02490c lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x891:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.044936:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc02490c lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x891:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.044958:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc02490c lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x891:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.045002:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520931.045016:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.045024:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc02490c lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x891:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.045097:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784ca00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.045102:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc02490c lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x891:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.045120:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc02490c lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x891:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.045138:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc02490c lrc: 1/0,0 mode: --/PW res: [0x280000401:0x891:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.045191:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc02490c lrc: 0/0,0 mode: --/PW res: [0x280000401:0x891:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.045255:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.045263:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.045270:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.045443:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515302680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+895:3626:x1870152993014400:12345-0@lo:6:osp-syn-0-0.0 Request processed in 3178us (3287us total) trans 4294977416 rc 0/0 00000100:00100000:3.0:1783520931.045467:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43934 00000004:00080000:3.0:1783520931.045513:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c554e00/2, rc 0, transno 4294977416 00000100:00100000:3.0:1783520931.045641:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c554e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993014400:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.045933:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515f08000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0000_UUID:35275:1870152993015552:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.045966:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024921 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x894:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.046031:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993016320 00000100:00100000:3.0:1783520931.046040:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993016320 00010000:00010000:3.0:1783520931.046170:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.046180:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000855d7374/0xd5401cbabc024905 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x890:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.046203:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfa400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.046210:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000855d7374/0xd5401cbabc024905 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x890:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.046223:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000855d7374/0xd5401cbabc024905 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x890:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.046249:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000855d7374/0xd5401cbabc024905 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x890:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.046272:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000855d7374/0xd5401cbabc024905 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x890:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.046313:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.046321:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.046328:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.046355:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.046499:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507939180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+137:2387:x1870153002328704:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 3098us (3801us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.046527:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35529 00000100:00100000:3.0:1783520931.046608:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993016448 00000100:00100000:3.0:1783520931.046619:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993016448 00000100:00100000:3.0:1783520931.046660:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43942 00000100:00100000:3.0:1783520931.046683:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ae4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+896:3625:x1870152993016448:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.046712:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2201 count 1 00010000:00010000:3.0:1783520931.046732:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024944 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x899:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.046773:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000afda7b9a/0xd5401cbabc022166 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x899:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f286b expref: 135 pid: 178496 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.046784:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x899:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.046790:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x899:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.046799:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024944 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x899:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.046912:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000afda7b9a/0xd5401cbabc022166 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x899:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f286b expref: 134 pid: 178496 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.046928:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000afda7b9a/0xd5401cbabc022166 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x899:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f286b expref: 134 pid: 178496 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.047004:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515f08000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0000_UUID:8408:1870152993016832:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.047116:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af1600 going to sleep for 11 seconds 00010000:00010000:3.0:1783520931.047139:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024921 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x894:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.047154:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024921 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x894:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.047172:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024921 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x894:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.047187:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024921 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x894:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.047203:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520931.047213:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.047220:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024921 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x894:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.047288:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784c200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.047293:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024921 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x894:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.047302:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024921 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x894:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.047321:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024921 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x894:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.047333:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc024921 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x894:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.047382:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.047386:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.047388:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.047546:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ae6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+898:3625:x1870152993015168:12345-0@lo:6:osp-syn-0-0.0 Request processed in 3193us (3741us total) trans 4294977419 rc 0/0 00000100:00100000:3.0:1783520931.047576:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43937 00000100:00100000:3.0:1783520931.047631:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35538 00000100:00100000:3.0:1783520931.047655:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507939880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+136:2387:x1870153002329728:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.047680:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.047693:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 0000000002d69dae/0xd5401cbabc02214a lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x897:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f284f expref: 136 pid: 35274 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.047707:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 0000000002d69dae/0xd5401cbabc02214a lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x897:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f284f expref: 136 pid: 35274 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.047731:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000002d69dae/0xd5401cbabc02214a lrc: 0/0,0 mode: --/PR res: [0x280000401:0x897:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f284f expref: 136 pid: 35274 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.047784:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x897:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.047790:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x897:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.047795:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc024936 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x897:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.047830:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc024936 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x897:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.047892:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.047927:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507939880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+133:2387:x1870153002329728:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 270us (692us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.048006:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35538 00010000:00010000:3.0:1783520931.048036:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.048044:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc024936 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x897:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.048082:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.048094:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc024936 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x897:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.048118:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc024936 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x897:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.048144:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc024936 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x897:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.048159:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc024936 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x897:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.048193:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.048197:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.048205:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.048242:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515f08000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0000_UUID:8408:1870152993016832:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.048274:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024944 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x899:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.048384:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993017344 00000100:00100000:3.0:1783520931.048431:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993017344 00000100:00100000:3.0:1783520931.048470:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43945 00000100:00100000:3.0:1783520931.048486:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d92680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+899:3625:x1870152993017344:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.048503:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2204 count 1 00000100:00100000:3.0:1783520931.048532:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d93800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993017472:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.048582:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520931.048640:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc024959 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x89c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.048688:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000790c2ccb/0xd5401cbabc02217b lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x89c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2895 expref: 131 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.048730:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x89c:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.048750:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x89c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.048772:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc024959 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x89c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.048831:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000790c2ccb/0xd5401cbabc02217b lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x89c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2895 expref: 131 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.048859:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000790c2ccb/0xd5401cbabc02217b lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x89c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2895 expref: 131 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.048879:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515f09f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0000_UUID:35274:1870152993017728:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.048944:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af1600 going to sleep for 11 seconds 00000100:00100000:3.0:1783520931.048984:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43947 00000100:00100000:3.0:1783520931.049001:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d90380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+901:3627:x1870152993017600:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.049016:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2206 count 1 00010000:00010000:3.0:1783520931.049030:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024967 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x89e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.049053:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000f82ad85e/0xd5401cbabc022189 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x89e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f28b1 expref: 130 pid: 178496 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.049070:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x89e:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.049074:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x89e:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.049079:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024967 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x89e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.049108:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000f82ad85e/0xd5401cbabc022189 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x89e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28b1 expref: 130 pid: 178496 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.049125:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000f82ad85e/0xd5401cbabc022189 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x89e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28b1 expref: 130 pid: 178496 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.049185:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515f0b100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0000_UUID:35275:1870152993017984:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.049221:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af10c0 going to sleep for 11 seconds 00010000:00010000:3.0:1783520931.049238:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024944 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x899:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.049248:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024944 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x899:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.049257:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024944 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x899:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.049265:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024944 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x899:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.049279:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520931.049290:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.049294:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024944 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x899:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.049303:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784c800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.049308:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024944 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x899:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.049316:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024944 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x899:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.049330:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024944 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x899:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.049677:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024944 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x899:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.049764:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.049769:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.049774:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.049964:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ae4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+903:3625:x1870152993016448:12345-0@lo:6:osp-syn-0-0.0 Request processed in 3284us (3458us total) trans 4294977424 rc 0/0 00000100:00100000:3.0:1783520931.049973:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43942 00000100:00100000:3.0:1783520931.050141:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35542 00000100:00100000:3.0:1783520931.050163:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e47b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+133:2388:x1870153002330368:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.050184:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.050203:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000958abacc/0xd5401cbabc022174 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x89b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2887 expref: 133 pid: 35275 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.050260:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000958abacc/0xd5401cbabc022174 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x89b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2887 expref: 133 pid: 35275 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.050289:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000958abacc/0xd5401cbabc022174 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x89b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2887 expref: 133 pid: 35275 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.050332:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x89b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.050356:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x89b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.050394:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000841b1b91/0xd5401cbabc024952 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x89b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.050406:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000841b1b91/0xd5401cbabc024952 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x89b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.050442:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.050483:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e47b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+130:2388:x1870153002330368:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 321us (471us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.050525:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35542 00010000:00010000:3.0:1783520931.050753:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.050831:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d92300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993018496:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.050911:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.051023:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515f09f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0000_UUID:35274:1870152993017728:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.051061:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc024959 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x89c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.051307:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515f0b100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0000_UUID:35275:1870152993017984:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.051342:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024967 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x89e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.051743:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35543 00000100:00100000:3.0:1783520931.051833:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507938380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+131:2390:x1870153002330624:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.051857:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.051870:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000f82ad85e/0xd5401cbabc022189 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x89e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28b1 expref: 131 pid: 178496 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.051915:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000f82ad85e/0xd5401cbabc022189 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x89e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28b1 expref: 131 pid: 178496 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.052012:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000f82ad85e/0xd5401cbabc022189 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x89e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28b1 expref: 129 pid: 178496 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.052071:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x89e:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.052135:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x89e:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.052147:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024967 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x89e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.052178:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024967 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x89e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.052230:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.052267:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507938380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+126:2390:x1870153002330624:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 439us (1011us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.052302:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35543 00000100:00100000:3.0:1783520931.052331:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002331008 00000100:00100000:3.0:1783520931.052339:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002331008 00010000:00010000:3.0:1783520931.052495:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 0000000046c2d14d/0xd5401cbabc022197 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x8a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28cd expref: 127 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.052555:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35548 00000100:00100000:3.0:1783520931.052578:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e47bb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+126:2387:x1870153002331008:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.052612:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.052627:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 0000000046c2d14d/0xd5401cbabc022197 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x8a0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28cd expref: 126 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.052657:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 0000000046c2d14d/0xd5401cbabc022197 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8a0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28cd expref: 125 pid: 9151 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.052676:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8a0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.052686:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8a0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.052692:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000c3fe08ef/0xd5401cbabc024975 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8a0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.052713:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000c3fe08ef/0xd5401cbabc024975 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x8a0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.052735:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.052742:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000c8b052a2/0xd5401cbabc024960 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x89d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.052754:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.052760:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000c8b052a2/0xd5401cbabc024960 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x89d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.052768:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000c8b052a2/0xd5401cbabc024960 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x89d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.052782:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000c8b052a2/0xd5401cbabc024960 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x89d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.052791:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c8b052a2/0xd5401cbabc024960 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x89d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.052879:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.052884:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.052893:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520931.052935:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d93800/2, rc 0, transno 4294977428 00000100:00100000:3.0:1783520931.052952:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d93800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993017472:0@lo:6:osp-syn-0-0.0 00010000:00010000:3.0:1783520931.053204:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.053211:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc02497c lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.053219:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.053224:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc02497c lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.053241:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc02497c lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.053265:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc02497c lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.053275:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc02497c lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.053581:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.053601:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.053607:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520931.053651:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d92300/2, rc 0, transno 4294977430 00000100:00100000:3.0:1783520931.053673:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d92300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993018496:0@lo:6:osp-syn-0-0.0 00010000:00010000:3.0:1783520931.053747:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024967 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x89e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.053763:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024967 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x89e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.053780:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024967 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x89e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.053796:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024967 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x89e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.053821:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520931.054152:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d90380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+910:3627:x1870152993017600:12345-0@lo:6:osp-syn-0-0.0 Request processed in 5153us (5301us total) trans 4294977431 rc 0/0 00000100:00100000:3.0:1783520931.054176:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43947 00010000:00010000:3.0:1783520931.054208:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.054253:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc024959 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x89c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.054304:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc024959 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x89c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.054313:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc024959 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x89c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.054323:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc024959 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x89c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.054645:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515f09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993020288:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.054685:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520931.054805:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520931.054871:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.055008:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d92680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+913:3625:x1870152993017344:12345-0@lo:6:osp-syn-0-0.0 Request processed in 6525us (6679us total) trans 4294977433 rc 0/0 00000100:00100000:3.0:1783520931.055036:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43945 00000004:00080000:3.0:1783520931.055087:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d91880/2, rc 0, transno 4294977433 00000100:00100000:3.0:1783520931.055098:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d91880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993017344:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.055173:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1783520931.055188:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515f08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993020800:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.055220:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.055275:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1783520931.055282:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515f0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993020672:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.055317:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.055392:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993020672 00000100:00100000:3.0:1783520931.055397:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993020672 00000100:00100000:3.0:1783520931.055425:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43959 00000100:00100000:3.0:1783520931.055437:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435130f6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+913:3625:x1870152993020672:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.055454:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2217 count 1 00010000:00010000:3.0:1783520931.055523:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc0249bb lrc: 3/0,1 mode: --/PW res: [0x280000401:0x8a9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.055575:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000b853b5a8/0xd5401cbabc022215 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x8a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f294b expref: 120 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.055583:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8a9:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.055601:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8a9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.055607:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc0249bb lrc: 4/0,1 mode: --/PW res: [0x280000401:0x8a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.055641:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000b853b5a8/0xd5401cbabc022215 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f294b expref: 120 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.055668:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000b853b5a8/0xd5401cbabc022215 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f294b expref: 119 pid: 20415 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.055696:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435130f4a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0000_UUID:35274:1870152993021056:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.055741:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff9436103d3300 going to sleep for 11 seconds 00010000:00010000:3.0:1783520931.055981:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520931.056168:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d91c00/2, rc 0, transno 4294977435 00000100:00100000:3.0:1783520931.056186:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d91c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993018368:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.056333:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993021184:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.056423:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:3.0:1783520931.056541:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515f0a300/2, rc 0, transno 4294977437 00000100:00100000:3.0:1783520931.056653:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515f0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993019904:0@lo:6:osp-syn-0-0.0 00010000:00010000:3.0:1783520931.056723:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.056733:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc024998 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.056765:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.056771:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc024998 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.056848:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc024998 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.056865:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc024998 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.056877:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc024998 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.056988:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.056995:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.056999:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.057078:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43962 00000100:00100000:3.0:1783520931.057094:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c550a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+916:3625:x1870152993021440:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.057115:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2221 count 1 00010000:00010000:3.0:1783520931.057131:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc0249d0 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x8ad:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.057176:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000c5e4d915/0xd5401cbabc022231 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x8ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2983 expref: 122 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.057198:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8ad:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.057205:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8ad:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.057220:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc0249d0 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x8ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.057279:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000c5e4d915/0xd5401cbabc022231 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2983 expref: 122 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.057338:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000c5e4d915/0xd5401cbabc022231 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2983 expref: 122 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.057499:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508059180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0000_UUID:35275:1870152993021824:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.057623:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af10c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520931.057656:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435130f4a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0000_UUID:35274:1870152993021056:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.057681:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc0249bb lrc: 3/0,1 mode: --/PW res: [0x280000401:0x8a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.057833:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520931.058344:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.058391:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc0249b4 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.058403:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.058411:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc0249b4 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.058452:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc0249b4 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.058475:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc0249b4 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.058486:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc0249b4 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.058512:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.058522:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.058533:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520931.058569:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515f08700/2, rc 0, transno 4294977438 00000100:00100000:3.0:1783520931.058583:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515f08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993020800:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.058664:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1783520931.058680:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435130f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993021952:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.058744:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:3.0:1783520931.058957:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515302680/2, rc 0, transno 4294977439 00000100:00100000:3.0:1783520931.058979:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515302680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993021184:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.059046:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1783520931.059062:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350793c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993022080:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.059197:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:3.0:1783520931.059243:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515f0b480/2, rc 0, transno 4294977441 00000100:00100000:3.0:1783520931.059251:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515f0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993020160:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.059430:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e46e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993022464:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.059504:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520931.059567:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.059604:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc0249bb lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.059630:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc0249bb lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.059643:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc0249bb lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.059653:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc0249bb lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.059719:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1783520931.059732:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350792ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993022720:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.059798:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520931.059837:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520931.059855:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.059872:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc0249bb lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.059914:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784c800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.059926:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc0249bb lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.059961:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc0249bb lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.060003:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc0249bb lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.060072:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc0249bb lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.060134:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.060139:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.060144:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.060508:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435130f6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+922:3625:x1870152993020672:12345-0@lo:6:osp-syn-0-0.0 Request processed in 5070us (5192us total) trans 4294977443 rc 0/0 00000100:00100000:3.0:1783520931.060517:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43959 00000100:00100000:3.0:1783520931.060852:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350792fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993023232:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.060893:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520931.061031:0:35275:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c5e4d915/0xd5401cbabc022231 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x8ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2983 expref: 111 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.061052:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508059180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0000_UUID:35275:1870152993021824:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.061069:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc0249d0 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.061081:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc0249d0 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.061137:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc0249d0 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.061178:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc0249d0 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.061183:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 0000000075ccb3c9/0xd5401cbabc0249d0 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.061206:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520931.061402:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c550a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+923:3625:x1870152993021440:12345-0@lo:6:osp-syn-0-0.0 Request processed in 4309us (4558us total) trans 4294977445 rc 0/0 00000100:00100000:3.0:1783520931.061417:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43962 00000100:00100000:3.0:1783520931.061506:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993023232 00000100:00100000:3.0:1783520931.061516:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993023232 00010000:00010000:3.0:1783520931.061564:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.061575:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc0249d7 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.061604:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.061614:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc0249d7 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.061623:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc0249d7 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.061639:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc0249d7 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.061652:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc0249d7 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.061678:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.061684:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.061690:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520931.061816:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435130f4a80/2, rc 0, transno 4294977446 00000100:00100000:3.0:1783520931.061834:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435130f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993021952:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.061945:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350792f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993023872:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.062001:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:3.0:1783520931.062062:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350793c700/2, rc 0, transno 4294977447 00000100:00100000:3.0:1783520931.062075:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350793c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993022080:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.062151:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993024128:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.062190:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520931.062228:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.062240:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000007aa431df/0xd5401cbabc0249de lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.062267:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.062274:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000007aa431df/0xd5401cbabc0249de lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.062297:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000007aa431df/0xd5401cbabc0249de lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.062324:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000007aa431df/0xd5401cbabc0249de lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.062425:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000007aa431df/0xd5401cbabc0249de lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.062467:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.062472:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.062475:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.062511:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993024128 00000100:00100000:3.0:1783520931.062523:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993024128 00000100:00100000:3.0:1783520931.062569:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43972 00000100:00100000:3.0:1783520931.062600:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+926:3628:x1870152993024128:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.062631:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2231 count 1 00010000:00010000:3.0:1783520931.062645:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024a16 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x8b7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.062711:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 000000006748f5fa/0xd5401cbabc022277 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x8b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a0f expref: 108 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.062748:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8b7:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.062754:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8b7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.062771:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024a16 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x8b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520931.062855:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350792ce00/2, rc 0, transno 4294977448 00000100:00100000:3.0:1783520931.062875:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350792ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993022720:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.062945:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1783520931.062965:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993024384:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.063020:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520931.063290:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 000000006748f5fa/0xd5401cbabc022277 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a0f expref: 104 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.063310:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 000000006748f5fa/0xd5401cbabc022277 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a0f expref: 104 pid: 12734 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.063321:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9d880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0000_UUID:35274:1870152993024640:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.063504:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af1600 going to sleep for 11 seconds 00010000:00010000:3.0:1783520931.063518:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.063695:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.063776:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c552680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993024896:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.063836:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.063941:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.064027:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.064577:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520931.064719:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.064727:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000007b280ca1/0xd5401cbabc024a01 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.064742:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.064748:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000007b280ca1/0xd5401cbabc024a01 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.064776:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000007b280ca1/0xd5401cbabc024a01 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.064810:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000007b280ca1/0xd5401cbabc024a01 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.064871:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000007b280ca1/0xd5401cbabc024a01 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.064984:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.064989:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.064996:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024a08 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.065039:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.065049:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024a08 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.065257:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024a08 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.065284:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024a08 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.065477:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024a08 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.065531:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.065543:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000001e04e98c/0xd5401cbabc024a0f lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.065555:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.065563:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000001e04e98c/0xd5401cbabc024a0f lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.065602:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000001e04e98c/0xd5401cbabc024a0f lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.065622:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000001e04e98c/0xd5401cbabc024a0f lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.065631:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000001e04e98c/0xd5401cbabc024a0f lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.065671:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.065677:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.065679:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.065713:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.065730:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993025536 00000100:00100000:3.0:1783520931.065736:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993025536 00000004:00080000:3.0:1783520931.065808:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350792f800/2, rc 0, transno 4294977453 00000100:00100000:3.0:1783520931.065831:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350792f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993023872:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.065924:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993025792:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.065999:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993025664 00000100:00100000:3.0:1783520931.066006:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993025664 00000100:00100000:3.0:1783520931.066025:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.066109:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43979 00000100:00100000:3.0:1783520931.066129:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350626e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+933:3626:x1870152993025664:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.066141:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2237 count 1 00010000:00010000:3.0:1783520931.066165:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024a47 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x8bd:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.066220:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000476b7781/0xd5401cbabc0222a1 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x8bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a63 expref: 101 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.066261:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8bd:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.066270:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8bd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.066281:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024a47 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x8bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.066344:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000476b7781/0xd5401cbabc0222a1 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a63 expref: 101 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.066387:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000476b7781/0xd5401cbabc0222a1 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a63 expref: 100 pid: 35282 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.066401:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9d500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0000_UUID:35275:1870152993026176:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.066459:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af10c0 going to sleep for 11 seconds 00010000:00010000:3.0:1783520931.066493:0:35274:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000006748f5fa/0xd5401cbabc022277 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x8b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a0f expref: 100 pid: 12734 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.066507:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9d880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0000_UUID:35274:1870152993024640:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.066531:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024a16 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.066540:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024a16 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.066545:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024a16 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.066553:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024a16 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.066570:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024a16 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.066602:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520931.066737:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+934:3628:x1870152993024128:12345-0@lo:6:osp-syn-0-0.0 Request processed in 4148us (4545us total) trans 4294977455 rc 0/0 00000100:00100000:3.0:1783520931.066910:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43972 00000004:00080000:3.0:1783520931.066946:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9f100/2, rc 0, transno 4294977455 00000100:00100000:3.0:1783520931.066963:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993024128:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.067029:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1783520931.067050:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993026304:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.067101:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.067167:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.067260:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993026304 00000100:00100000:3.0:1783520931.067271:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993026304 00010000:00010000:3.0:1783520931.067418:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.067426:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000a2269f0c/0xd5401cbabc024a1d lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.067435:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.067439:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000a2269f0c/0xd5401cbabc024a1d lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.067448:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000a2269f0c/0xd5401cbabc024a1d lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.067471:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000a2269f0c/0xd5401cbabc024a1d lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.067501:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000a2269f0c/0xd5401cbabc024a1d lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.067543:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.067553:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.067557:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.067703:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43981 00000100:00100000:3.0:1783520931.067727:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+935:3627:x1870152993026304:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.067755:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2239 count 1 00010000:00010000:3.0:1783520931.067779:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024a55 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x8bf:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.067798:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 000000001f7e0a8e/0xd5401cbabc0222af lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x8bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a7f expref: 98 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.067927:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8bf:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.067933:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8bf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.067938:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024a55 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x8bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.067972:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 000000001f7e0a8e/0xd5401cbabc0222af lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a7f expref: 98 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.068006:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 000000001f7e0a8e/0xd5401cbabc0222af lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a7f expref: 98 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.068020:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9ca80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0000_UUID:35274:1870152993026944:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.068072:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af1600 going to sleep for 11 seconds 00000100:00100000:3.0:1783520931.068132:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.068250:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.068466:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.068635:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520931.068651:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.068659:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000005f11335d/0xd5401cbabc024a39 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.068669:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.068682:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000005f11335d/0xd5401cbabc024a39 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.068725:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000005f11335d/0xd5401cbabc024a39 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.068827:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000005f11335d/0xd5401cbabc024a39 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.068854:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000005f11335d/0xd5401cbabc024a39 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.068932:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.068939:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.068945:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.069047:0:35275:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000476b7781/0xd5401cbabc0222a1 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x8bd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a63 expref: 97 pid: 35282 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.069076:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9d500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0000_UUID:35275:1870152993026176:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.069108:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024a47 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x8bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.069235:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024a47 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.069274:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024a47 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.069327:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024a47 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.069355:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024a47 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.069400:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520931.069504:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350626e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+938:3626:x1870152993025664:12345-0@lo:6:osp-syn-0-0.0 Request processed in 3376us (3521us total) trans 4294977459 rc 0/0 00000100:00100000:3.0:1783520931.069515:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43979 00010000:00010000:3.0:1783520931.069714:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.069721:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc024a32 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.069735:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.069743:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc024a32 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.069776:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc024a32 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.069810:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc024a32 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.069820:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000005461c69f/0xd5401cbabc024a32 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.069847:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.069850:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.069853:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520931.069885:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9bb80/2, rc 0, transno 4294977461 00000100:00100000:3.0:1783520931.069905:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993025280:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.070005:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520931.070129:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.070139:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc024a40 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.070163:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.070169:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc024a40 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.070186:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc024a40 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.070266:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc024a40 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.070276:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc024a40 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.070309:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.070318:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.070324:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.070483:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1783520931.070502:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993027968:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.070566:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.070883:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.071020:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.071087:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.071124:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002335232 00000100:00100000:3.0:1783520931.071133:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002335232 00010000:00010000:3.0:1783520931.071158:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000005935c58e/0xd5401cbabc0222c4 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a9b expref: 94 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.071213:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002335360 00000100:00100000:3.0:1783520931.071220:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002335360 00010000:00010000:3.0:1783520931.071238:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000009d8054f6/0xd5401cbabc0222cb lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x8c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2aa9 expref: 95 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.071447:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35579 00000100:00100000:3.0:1783520931.071495:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9e680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+95:2387:x1870153002335104:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.071521:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.071537:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000001f7e0a8e/0xd5401cbabc0222af lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x8bf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a7f expref: 95 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.071570:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000001f7e0a8e/0xd5401cbabc0222af lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8bf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a7f expref: 95 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.071618:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8bf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.071625:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8bf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.071638:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024a55 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8bf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.071720:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024a55 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x8bf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.071744:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.071784:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9e680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+94:2387:x1870153002335104:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 318us (908us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.071891:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35579 00010000:00010000:3.0:1783520931.071997:0:35274:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000001f7e0a8e/0xd5401cbabc0222af lrc: 0/0,0 mode: --/PR res: [0x280000401:0x8bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a7f expref: 89 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.072017:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9ca80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0000_UUID:35274:1870152993026944:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.072088:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024a55 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.072100:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024a55 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.072107:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024a55 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.072116:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024a55 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.072124:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024a55 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.072141:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520931.072152:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.072158:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024a55 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.072172:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784cc00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.072177:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024a55 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.072185:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024a55 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.072196:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024a55 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.072204:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024a55 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.072224:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.072229:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc024a63 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.072245:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.072253:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc024a63 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.072262:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc024a63 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.072275:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc024a63 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.072283:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000007d3e164/0xd5401cbabc024a63 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.072299:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.072302:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.072305:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.072315:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.072543:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+944:3627:x1870152993026304:12345-0@lo:6:osp-syn-0-0.0 Request processed in 4822us (5451us total) trans 4294977466 rc 0/0 00000100:00100000:3.0:1783520931.072554:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43981 00000004:00080000:3.0:1783520931.072608:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9d880/2, rc 0, transno 4294977466 00000100:00100000:3.0:1783520931.072623:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993026304:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.072664:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35582 00000100:00100000:3.0:1783520931.072702:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507938000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+91:2387:x1870153002335616:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.072714:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.072724:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000de2c21c5/0xd5401cbabc0222d2 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x8c3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ab7 expref: 91 pid: 35282 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.072757:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000de2c21c5/0xd5401cbabc0222d2 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8c3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ab7 expref: 91 pid: 35282 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.072791:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8c3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.072804:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8c3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.072822:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000007aa431df/0xd5401cbabc024a6a lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8c3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.072866:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000007aa431df/0xd5401cbabc024a6a lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x8c3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.072912:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.072939:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507938000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+91:2387:x1870153002335616:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 252us (793us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.072973:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35582 00010000:00010000:3.0:1783520931.073421:0:9151:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000de2c21c5/0xd5401cbabc0222d2 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x8c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ab7 expref: 87 pid: 35282 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.073437:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c5f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0000_UUID:9151:1870152993027840:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.073454:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000007aa431df/0xd5401cbabc024a6a lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.073464:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000007aa431df/0xd5401cbabc024a6a lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.073473:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000007aa431df/0xd5401cbabc024a6a lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.073482:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000007aa431df/0xd5401cbabc024a6a lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.073489:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000007aa431df/0xd5401cbabc024a6a lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.073506:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520931.073585:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+947:3627:x1870152993027328:12345-0@lo:6:osp-syn-0-0.0 Request processed in 3841us (3932us total) trans 4294977468 rc 0/0 00000100:00100000:3.0:1783520931.073605:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43984 00000004:00080000:3.0:1783520931.073696:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9aa00/2, rc 0, transno 4294977468 00000100:00100000:3.0:1783520931.073710:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993027328:0@lo:6:osp-syn-0-0.0 00010000:00010000:3.0:1783520931.073792:0:35275:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000ee6644eb/0xd5401cbabc0222e0 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x8c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ad3 expref: 86 pid: 35274 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.073808:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c551180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0000_UUID:35275:1870152993028096:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.073823:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024a78 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.073832:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024a78 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.073840:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024a78 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.073848:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024a78 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.073855:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024a78 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.074142:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520931.074152:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.074156:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024a78 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.074164:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.074209:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024a78 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.074286:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024a78 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.074297:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024a78 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.074303:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024a78 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.074314:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.074315:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.074316:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.074336:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.074443:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.074505:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.074543:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35586 00000100:00100000:3.0:1783520931.074609:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+87:2388:x1870153002335872:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.074623:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.074636:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 0000000067aa986b/0xd5401cbabc0222e7 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8c6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ae1 expref: 87 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.074676:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 0000000067aa986b/0xd5401cbabc0222e7 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8c6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ae1 expref: 87 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.074706:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000067aa986b/0xd5401cbabc0222e7 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x8c6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ae1 expref: 87 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.074730:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8c6:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.074746:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8c6:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.074764:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000ddf44b9d/0xd5401cbabc024a7f lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.074792:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000ddf44b9d/0xd5401cbabc024a7f lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x8c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.074841:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520931.074857:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.074865:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc024a8d lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.074886:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.074897:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc024a8d lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.074943:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc024a8d lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.074957:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc024a8d lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.074968:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc024a8d lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.074994:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.074996:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.075002:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.075182:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+948:3626:x1870152993027584:12345-0@lo:6:osp-syn-0-0.0 Request processed in 4704us (5048us total) trans 4294977471 rc 0/0 00000100:00100000:3.0:1783520931.075191:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43986 00000100:00100000:3.0:1783520931.075236:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+83:2388:x1870153002335872:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 680us (874us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.075244:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35586 00000100:00100000:3.0:1783520931.075963:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.076007:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.076054:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.076084:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.076316:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35588 00000100:00100000:3.0:1783520931.076336:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94361b4a6050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+86:2388:x1870153002336256:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.076349:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.076450:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000d579e961/0xd5401cbabc022303 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8ca:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b19 expref: 86 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.076539:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000d579e961/0xd5401cbabc022303 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8ca:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b19 expref: 86 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.076644:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000d579e961/0xd5401cbabc022303 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x8ca:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b19 expref: 85 pid: 35276 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.076686:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8ca:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.076743:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8ca:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.076763:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc024a94 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.076824:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc024a94 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x8ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.076886:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.076920:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94361b4a6050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+82:2388:x1870153002336256:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 588us (952us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.076944:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35588 00010000:00010000:3.0:1783520931.076988:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.076995:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000dc1a697e/0xd5401cbabc024aa2 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.077004:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.077008:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000dc1a697e/0xd5401cbabc024aa2 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.077019:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000dc1a697e/0xd5401cbabc024aa2 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.077047:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000dc1a697e/0xd5401cbabc024aa2 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.077062:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000dc1a697e/0xd5401cbabc024aa2 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.077114:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.077119:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.077125:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.077174:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.077178:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000005120a6/0xd5401cbabc024aa9 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.077186:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.077189:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000005120a6/0xd5401cbabc024aa9 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.077196:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000005120a6/0xd5401cbabc024aa9 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.077208:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000005120a6/0xd5401cbabc024aa9 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.077220:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000005120a6/0xd5401cbabc024aa9 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.077238:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.077242:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc024a9b lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.077249:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.077252:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc024a9b lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.077260:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc024a9b lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.077279:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc024a9b lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.077290:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000d4fdb58b/0xd5401cbabc024a9b lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.077412:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.077416:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.077465:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.077478:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993030912 00000100:00100000:3.0:1783520931.077483:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993030912 00000100:00100000:3.0:1783520931.077527:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 43998 00000100:00100000:3.0:1783520931.077543:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e450700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+952:3625:x1870152993030912:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.077561:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2257 count 1 00010000:00010000:3.0:1783520931.077577:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024acc lrc: 3/0,1 mode: --/PW res: [0x280000401:0x8d1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.077610:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 0000000042f1ea6c/0xd5401cbabc022342 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x8d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b7b expref: 79 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.077618:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8d1:0x0].0x0 since=32s v=(0 0) 00010000:00010000:3.0:1783520931.077627:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8d1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.077635:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024acc lrc: 4/0,1 mode: --/PW res: [0x280000401:0x8d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.077677:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 0000000042f1ea6c/0xd5401cbabc022342 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b7b expref: 80 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.077691:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 0000000042f1ea6c/0xd5401cbabc022342 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b7b expref: 80 pid: 20140 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.077724:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9ca80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0000_UUID:9151:1870152993031168:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.077779:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512af10c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520931.077847:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.077930:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.078136:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002337024 00000100:00100000:3.0:1783520931.078147:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002337024 00010000:00010000:3.0:1783520931.078173:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000e5f2f058/0xd5401cbabc02232d lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b6d expref: 80 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.078259:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35595 00000100:00100000:3.0:1783520931.078280:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f5850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+80:2387:x1870153002337024:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.078292:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.078304:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000e5f2f058/0xd5401cbabc02232d lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8d0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b6d expref: 80 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.078347:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000e5f2f058/0xd5401cbabc02232d lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8d0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b6d expref: 79 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.078492:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000e5f2f058/0xd5401cbabc02232d lrc: 0/0,0 mode: --/PR res: [0x280000401:0x8d0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b6d expref: 78 pid: 8410 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.078515:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8d0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.078520:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8d0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.078525:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000005cc63f31/0xd5401cbabc024ac5 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.078533:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000005cc63f31/0xd5401cbabc024ac5 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x8d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.078557:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.078606:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f5850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+77:2387:x1870153002337024:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 330us (686us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.078620:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35595 00010000:00010000:3.0:1783520931.078751:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.078758:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 0000000018903c53/0xd5401cbabc024ab7 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.078767:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.078773:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 0000000018903c53/0xd5401cbabc024ab7 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.078786:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 0000000018903c53/0xd5401cbabc024ab7 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.078813:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 0000000018903c53/0xd5401cbabc024ab7 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.078826:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000018903c53/0xd5401cbabc024ab7 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.078853:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.078857:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.078860:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.079041:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.079214:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9ca80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0000_UUID:9151:1870152993031168:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.079297:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024acc lrc: 3/0,1 mode: --/PW res: [0x280000401:0x8d1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.079353:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44003 00010000:00010000:3.0:1783520931.079411:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.079420:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc024a94 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.079443:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.079452:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc024a94 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.079461:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc024a94 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.079478:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc024a94 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.079491:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc024a94 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.079519:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.079522:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.079529:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.079546:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024acc lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.079564:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024acc lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.079580:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024acc lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.079607:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024acc lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.079620:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520931.079664:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520931.079669:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024acc lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.079676:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784de00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.079682:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024acc lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.079695:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024acc lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.079708:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024acc lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.079715:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024acc lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.079790:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.079793:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.079795:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.079855:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e450700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+959:3625:x1870152993030912:12345-0@lo:6:osp-syn-0-0.0 Request processed in 2313us (2394us total) trans 4294977480 rc 0/0 00000100:00100000:3.0:1783520931.079870:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 43998 00000100:00100000:3.0:1783520931.079895:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+958:3628:x1870152993031936:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.079913:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2262 count 1 00010000:00010000:3.0:1783520931.079981:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc024af6 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x8d6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.079999:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 000000008c4076de/0xd5401cbabc022357 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x8d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bc1 expref: 73 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.080005:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8d6:0x0].0x0 since=33s v=(0 0) 00010000:00010000:3.0:1783520931.080008:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8d6:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.080013:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc024af6 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x8d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.080032:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 000000008c4076de/0xd5401cbabc022357 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bc1 expref: 73 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.080045:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 000000008c4076de/0xd5401cbabc022357 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bc1 expref: 73 pid: 8410 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.080056:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9ca80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0000_UUID:35275:1870152993032704:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.080083:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514825600 going to sleep for 11 seconds 00000100:00100000:3.0:1783520931.080494:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.080604:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d98000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0000_UUID:20140:1870152993031680:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.080619:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 0000000066426cc1/0xd5401cbabc024ae1 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x8d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.080694:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.080719:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002337408 00000100:00100000:3.0:1783520931.080724:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002337408 00010000:00010000:3.0:1783520931.080789:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000009d2fa220/0xd5401cbabc02233b lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b97 expref: 75 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.080837:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35598 00000100:00100000:3.0:1783520931.080855:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+75:2387:x1870153002337408:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.080871:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.080880:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 1s ns: filter-lustre-OST0000_UUID lock: 000000009d2fa220/0xd5401cbabc02233b lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8d3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b97 expref: 75 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.080938:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000009d2fa220/0xd5401cbabc02233b lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8d3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b97 expref: 74 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.080997:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000009d2fa220/0xd5401cbabc02233b lrc: 0/0,0 mode: --/PR res: [0x280000401:0x8d3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b97 expref: 74 pid: 8408 timeout: 4225 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.081015:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8d3:0x0].0x0 since=1s v=(0 0) 00010000:00010000:3.0:1783520931.081019:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8d3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.081024:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc024ada lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.081124:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc024ada lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x8d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.081238:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.081326:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+72:2387:x1870153002337408:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 475us (833us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.081339:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35598 00000100:00100000:3.0:1783520931.081520:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.081570:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.081648:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520931.081788:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.081802:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc024ada lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.081819:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.081829:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc024ada lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.081886:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc024ada lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.082009:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc024ada lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.082032:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000009eb77304/0xd5401cbabc024ada lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.082091:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.082096:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.082099:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.082129:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9ca80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0000_UUID:35275:1870152993032704:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.082149:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc024af6 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x8d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.082228:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.082238:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000e3859524/0xd5401cbabc024ad3 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.082249:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.082257:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000e3859524/0xd5401cbabc024ad3 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.082273:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000e3859524/0xd5401cbabc024ad3 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.082293:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000e3859524/0xd5401cbabc024ad3 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.082338:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000e3859524/0xd5401cbabc024ad3 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.082402:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.082406:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.082412:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.082551:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.082605:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.082661:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.082693:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002338304 00000100:00100000:3.0:1783520931.082699:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002338304 00010000:00010000:3.0:1783520931.082724:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 0000000013affd72/0xd5401cbabc022365 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x8d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bdd expref: 73 pid: 35276 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.082806:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35602 00000100:00100000:3.0:1783520931.082824:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f0050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+73:2388:x1870153002338176:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.082835:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.082847:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 0000000052af5769/0xd5401cbabc02236c lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8d9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2beb expref: 73 pid: 9151 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.082906:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 0000000052af5769/0xd5401cbabc02236c lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8d9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2beb expref: 73 pid: 9151 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.082974:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000052af5769/0xd5401cbabc02236c lrc: 0/0,0 mode: --/PR res: [0x280000401:0x8d9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2beb expref: 73 pid: 9151 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.083029:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8d9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.083037:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8d9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.083056:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000a2269f0c/0xd5401cbabc024afd lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.083117:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000a2269f0c/0xd5401cbabc024afd lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x8d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.083168:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.083194:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f0050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+70:2388:x1870153002338176:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 372us (658us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.083222:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35602 00010000:00010000:3.0:1783520931.083285:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc024af6 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.083330:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc024af6 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.083412:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc024af6 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.083468:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000000d04a675/0xd5401cbabc024af6 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.083487:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:3.0:1783520931.083571:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+964:3628:x1870152993031936:12345-0@lo:6:osp-syn-0-0.0 Request processed in 4175us (4358us total) trans 4294977486 rc 0/0 00000100:00100000:3.0:1783520931.083599:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44003 00010000:00010000:3.0:1783520931.083892:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.083986:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993034112 00000100:00100000:3.0:1783520931.083990:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993034112 00000100:00100000:3.0:1783520931.084036:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44010 00000100:00100000:3.0:1783520931.084054:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d92d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+965:3627:x1870152993034112:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.084074:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2270 count 1 00010000:00010000:3.0:1783520931.084090:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024b20 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x8de:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.084107:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 0000000079cc0aea/0xd5401cbabc02238f lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x8de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c31 expref: 66 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.084122:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8de:0x0].0x0 since=33s v=(0 0) 00010000:00010000:3.0:1783520931.084125:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8de:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.084131:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024b20 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x8de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.084157:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 0000000079cc0aea/0xd5401cbabc02238f lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c31 expref: 66 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.084170:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 0000000079cc0aea/0xd5401cbabc02238f lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c31 expref: 66 pid: 8409 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.084191:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9d180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0000_UUID:9151:1870152993034240:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.084230:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514825600 going to sleep for 11 seconds 00010000:00010000:3.0:1783520931.084322:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.084328:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000841b1b91/0xd5401cbabc024aef lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.084426:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.084432:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000841b1b91/0xd5401cbabc024aef lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.084441:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000841b1b91/0xd5401cbabc024aef lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.084477:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000841b1b91/0xd5401cbabc024aef lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.084523:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000841b1b91/0xd5401cbabc024aef lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.084556:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.084561:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.084564:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.086815:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9d180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0000_UUID:9151:1870152993034240:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.086938:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024b20 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x8de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.087055:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.087078:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.087103:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.087191:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.087225:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.087258:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.087289:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.087324:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.087343:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002338944 00000100:00100000:3.0:1783520931.087351:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002338944 00010000:00010000:3.0:1783520931.087397:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000e23a9c85/0xd5401cbabc022396 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c3f expref: 71 pid: 35276 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.087420:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35605 00000100:00100000:3.0:1783520931.087432:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+73:2388:x1870153002338560:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.087445:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.087461:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000007a4fc52d/0xd5401cbabc02237a lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8db:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c07 expref: 73 pid: 12734 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.087492:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000007a4fc52d/0xd5401cbabc02237a lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8db:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c07 expref: 73 pid: 12734 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.087522:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000007a4fc52d/0xd5401cbabc02237a lrc: 0/0,0 mode: --/PR res: [0x280000401:0x8db:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c07 expref: 73 pid: 12734 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.087547:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8db:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.087569:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8db:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.087579:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000007b280ca1/0xd5401cbabc024b12 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.087611:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000007b280ca1/0xd5401cbabc024b12 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x8db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.087671:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.087719:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+71:2388:x1870153002338560:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 281us (656us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.087752:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35605 00000100:00100000:3.0:1783520931.087779:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35608 00000100:00100000:3.0:1783520931.087799:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9e680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+69:2387:x1870153002338688:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.087822:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.087834:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 0000000079cc0aea/0xd5401cbabc02238f lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8de:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c31 expref: 69 pid: 8409 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.087887:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 0000000079cc0aea/0xd5401cbabc02238f lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8de:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c31 expref: 69 pid: 8409 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.087950:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000079cc0aea/0xd5401cbabc02238f lrc: 0/0,0 mode: --/PR res: [0x280000401:0x8de:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c31 expref: 69 pid: 8409 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.087963:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8de:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.087967:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8de:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.087971:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024b20 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.087976:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024b20 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x8de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.087995:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520931.088002:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000007b280ca1/0xd5401cbabc024b12 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.088016:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.088023:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000007b280ca1/0xd5401cbabc024b12 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.088032:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000007b280ca1/0xd5401cbabc024b12 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.088043:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000007b280ca1/0xd5401cbabc024b12 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.088048:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000007b280ca1/0xd5401cbabc024b12 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.088067:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.088079:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024b0b lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.088094:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.088103:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024b0b lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.088176:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024b0b lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.088235:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024b0b lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.088245:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000044471100/0xd5401cbabc024b0b lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.088301:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.088306:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.088313:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.088329:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.088355:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9e680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+65:2387:x1870153002338688:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 557us (1162us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.088380:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35608 00000100:00100000:3.0:1783520931.088407:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35611 00000100:00100000:3.0:1783520931.088422:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9f800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+64:2387:x1870153002339200:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.088439:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.088447:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 0000000096baa45d/0xd5401cbabc022388 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8dd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c23 expref: 64 pid: 8409 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.088463:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 0000000096baa45d/0xd5401cbabc022388 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8dd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c23 expref: 64 pid: 8409 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.088523:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000096baa45d/0xd5401cbabc022388 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x8dd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c23 expref: 63 pid: 8409 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.088533:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8dd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.088544:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8dd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.088551:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc024b27 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.088603:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc024b27 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x8dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.088649:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.088677:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9f800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+60:2387:x1870153002339200:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 256us (1349us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.088709:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35611 00010000:00010000:3.0:1783520931.088732:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.088737:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc024b19 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.088747:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.088757:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc024b19 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.088790:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc024b19 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.088821:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc024b19 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.088877:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc024b19 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.088907:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.088910:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.088913:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.088926:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024b20 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.088932:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024b20 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.088942:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024b20 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.088989:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024b20 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.089018:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520931.089053:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520931.089065:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024b20 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.089135:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784cc00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.089144:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024b20 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.089178:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024b20 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.089219:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024b20 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.089268:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024b20 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.089283:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.089293:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc024b27 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.089308:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.089312:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc024b27 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.089329:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc024b27 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.089342:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc024b27 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.089433:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c67c07c2/0xd5401cbabc024b27 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.089451:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.089455:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 0000000000a37bb7/0xd5401cbabc024b3c lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.089471:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.089476:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 0000000000a37bb7/0xd5401cbabc024b3c lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.089483:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 0000000000a37bb7/0xd5401cbabc024b3c lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.089500:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 0000000000a37bb7/0xd5401cbabc024b3c lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.089507:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000000a37bb7/0xd5401cbabc024b3c lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.089521:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.089527:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.089531:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.089542:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.089649:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d92d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+970:3627:x1870152993034112:12345-0@lo:6:osp-syn-0-0.0 Request processed in 5598us (5687us total) trans 4294977496 rc 0/0 00000100:00100000:3.0:1783520931.089663:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44010 00000100:00100000:3.0:1783520931.090696:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993036544 00000100:00100000:3.0:1783520931.090706:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993036544 00000100:00100000:3.0:1783520931.091897:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.091925:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.092069:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.092106:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.092255:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.092296:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121b0380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0000_UUID:178496:1870152993036288:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.092347:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc024b51 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x8e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.092410:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35614 00000100:00100000:3.0:1783520931.092431:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+62:2390:x1870153002339712:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.092443:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.092455:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000b736352e/0xd5401cbabc0223ce lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8e7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2caf expref: 62 pid: 35274 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.092558:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000b736352e/0xd5401cbabc0223ce lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8e7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2caf expref: 62 pid: 35274 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.092727:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000b736352e/0xd5401cbabc0223ce lrc: 0/0,0 mode: --/PR res: [0x280000401:0x8e7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2caf expref: 60 pid: 35274 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.092773:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8e7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.092778:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8e7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.092784:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc024b51 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.092798:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc024b51 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x8e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.092877:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520931.092902:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc024b51 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.092959:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc024b51 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.092976:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc024b51 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.092997:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc024b51 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.093043:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:3.0:1783520931.093140:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d98700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+977:3628:x1870152993036160:12345-0@lo:6:osp-syn-0-0.0 Request processed in 2941us (3037us total) trans 4294977499 rc 0/0 00000100:00100000:3.0:1783520931.093152:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44017 00010000:00010000:3.0:1783520931.093174:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520931.093180:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc024b51 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.093207:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.093211:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc024b51 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.093234:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc024b51 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.093250:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc024b51 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.093287:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc024b51 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.093324:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.093329:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.093336:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.093346:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.093659:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+54:2390:x1870153002339712:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 1235us (1586us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.093740:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35614 00010000:00010000:3.0:1783520931.093883:0:20415:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000ccb622bb/0xd5401cbabc0223b2 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x8e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c77 expref: 53 pid: 178496 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.093952:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121b2300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0000_UUID:20415:1870152993035776:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.093980:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000005cc63f31/0xd5401cbabc024b4a lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.094020:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000005cc63f31/0xd5401cbabc024b4a lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.094051:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000005cc63f31/0xd5401cbabc024b4a lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.094057:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000005cc63f31/0xd5401cbabc024b4a lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.094061:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000005cc63f31/0xd5401cbabc024b4a lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.094076:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520931.094085:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.094102:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 000000005cc63f31/0xd5401cbabc024b4a lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.094130:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.094141:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 000000005cc63f31/0xd5401cbabc024b4a lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.094226:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 000000005cc63f31/0xd5401cbabc024b4a lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.094241:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 000000005cc63f31/0xd5401cbabc024b4a lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.094250:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000005cc63f31/0xd5401cbabc024b4a lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.094276:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.094281:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.094285:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520931.094628:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121b3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+979:3628:x1870152993035520:12345-0@lo:6:osp-syn-0-0.0 Request processed in 4683us (4770us total) trans 4294977502 rc 0/0 00000100:00100000:3.0:1783520931.094635:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44016 00000100:00100000:3.0:1783520931.094672:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993037696 00000100:00100000:3.0:1783520931.094679:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993037696 00000100:00100000:3.0:1783520931.094726:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993037568 00000100:00100000:3.0:1783520931.094734:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993037568 00000100:00100000:3.0:1783520931.094767:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44025 00000100:00100000:3.0:1783520931.094781:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350793a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+980:3626:x1870152993037696:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.094796:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2285 count 1 00010000:00010000:3.0:1783520931.094811:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024b89 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x8ed:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.094833:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000168629d0/0xd5401cbabc0223f8 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x8ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d03 expref: 55 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.094846:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8ed:0x0].0x0 since=33s v=(0 0) 00010000:00010000:3.0:1783520931.094850:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8ed:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.094858:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024b89 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x8ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.094933:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000168629d0/0xd5401cbabc0223f8 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d03 expref: 55 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.094971:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000168629d0/0xd5401cbabc0223f8 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d03 expref: 55 pid: 35274 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.094986:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9ea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0000_UUID:178496:1870152993038080:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.095029:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350d1ce000 going to sleep for 11 seconds 00000100:00100000:3.0:1783520931.095108:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44026 00000100:00100000:3.0:1783520931.095130:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350793b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+980:3626:x1870152993037568:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.095197:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2284 count 1 00010000:00010000:3.0:1783520931.095214:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024b90 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x8ec:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.095257:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 0000000000663c41/0xd5401cbabc0223f1 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x8ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cf5 expref: 53 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.095291:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8ec:0x0].0x0 since=33s v=(0 0) 00010000:00010000:3.0:1783520931.095299:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8ec:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.095305:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024b90 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x8ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.095442:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520931.095449:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024b6d lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.095457:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520931.095462:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024b6d lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.095520:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024b6d lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x8e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.095544:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024b6d lrc: 1/0,0 mode: --/PW res: [0x280000401:0x8e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.095553:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000c1f24f65/0xd5401cbabc024b6d lrc: 0/0,0 mode: --/PW res: [0x280000401:0x8e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.095575:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.095580:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.095597:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520931.095637:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 0000000000663c41/0xd5401cbabc0223f1 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cf5 expref: 50 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.095652:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 0000000000663c41/0xd5401cbabc0223f1 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cf5 expref: 49 pid: 8409 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.095673:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9f800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0000_UUID:20415:1870152993038336:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.095766:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350a20d540 going to sleep for 11 seconds 00000100:00100000:3.0:1783520931.097163:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9ea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0000_UUID:178496:1870152993038080:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.097221:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024b89 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x8ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.098464:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9f800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0000_UUID:20415:1870152993038336:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.098488:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000009f4b4cf1/0xd5401cbabc024b90 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x8ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.098541:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002340480 00000100:00100000:3.0:1783520931.098547:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002340480 00010000:00010000:3.0:1783520931.098571:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 0000000059c8dff3/0xd5401cbabc0223ea lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ce7 expref: 50 pid: 20140 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.098622:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002340736 00000100:00100000:3.0:1783520931.098627:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002340736 00010000:00010000:3.0:1783520931.098640:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 0000000000663c41/0xd5401cbabc0223f1 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cf5 expref: 51 pid: 8409 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.098678:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002340608 00000100:00100000:3.0:1783520931.098684:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002340608 00010000:00010000:3.0:1783520931.098707:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000168629d0/0xd5401cbabc0223f8 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d03 expref: 53 pid: 35274 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.098745:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35621 00000100:00100000:3.0:1783520931.098764:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121b1500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+53:2388:x1870153002340480:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.098779:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.098789:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 0000000059c8dff3/0xd5401cbabc0223ea lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8eb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ce7 expref: 53 pid: 20140 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.098810:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 0000000059c8dff3/0xd5401cbabc0223ea lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8eb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ce7 expref: 53 pid: 20140 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.098855:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000059c8dff3/0xd5401cbabc0223ea lrc: 0/0,0 mode: --/PR res: [0x280000401:0x8eb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ce7 expref: 53 pid: 20140 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.098891:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8eb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.098898:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8eb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.098912:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc024b82 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.098967:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc024b82 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x8eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.098996:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.099044:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121b1500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+50:2388:x1870153002340480:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 288us (554us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.099055:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35621 00000100:00100000:3.0:1783520931.099094:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35624 00000100:00100000:3.0:1783520931.099108:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121b3800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+48:2387:x1870153002340864:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.099119:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.099126:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000cd104a24/0xd5401cbabc0223ff lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x8ee:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d11 expref: 47 pid: 35275 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.099142:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000cd104a24/0xd5401cbabc0223ff lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8ee:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d11 expref: 47 pid: 35275 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.099166:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8ee:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.099170:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8ee:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.099183:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000c1a73293/0xd5401cbabc024b97 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8ee:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.099200:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000c1a73293/0xd5401cbabc024b97 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x8ee:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.099220:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.099249:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121b3800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+46:2387:x1870153002340864:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 145us (589us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.099261:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35624 00010000:00010000:3.0:1783520931.099293:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024b89 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.099303:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024b89 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.099319:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024b89 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.099327:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000001ccbdf45/0xd5401cbabc024b89 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.099345:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:3.0:1783520931.099536:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350793a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+985:3626:x1870152993037696:12345-0@lo:6:osp-syn-0-0.0 Request processed in 4754us (5037us total) trans 4294977507 rc 0/0 00000100:00100000:3.0:1783520931.099565:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44025 00000004:00080000:3.0:1783520931.099636:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507939f80/2, rc 0, transno 4294977507 00000100:00100000:3.0:1783520931.099646:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507939f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993037696:0@lo:6:osp-syn-0-0.0 00000004:00080000:3.0:1783520931.100463:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507938e00/2, rc 0, transno 4294977509 00000100:00100000:3.0:1783520931.100486:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507938e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993037568:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.100577:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993040128:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.100682:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.101188:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002341248 00000100:00100000:3.0:1783520931.101194:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002341248 00010000:00010000:3.0:1783520931.101214:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000d8e3e60d/0xd5401cbabc022414 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d3b expref: 47 pid: 12734 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.101243:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002341376 00000100:00100000:3.0:1783520931.101249:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002341376 00010000:00010000:3.0:1783520931.101268:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 0000000042ef1cd5/0xd5401cbabc0223e3 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x8ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cd9 expref: 49 pid: 8409 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.101291:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35626 00000100:00100000:3.0:1783520931.101304:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350792ca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+49:2388:x1870153002341248:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.101318:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.101325:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000d8e3e60d/0xd5401cbabc022414 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8f1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d3b expref: 49 pid: 12734 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.101347:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000d8e3e60d/0xd5401cbabc022414 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8f1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d3b expref: 49 pid: 12734 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.101426:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000d8e3e60d/0xd5401cbabc022414 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x8f1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d3b expref: 49 pid: 12734 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.101451:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8f1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.101455:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8f1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.101465:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc024bac lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.101478:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc024bac lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x8f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.101513:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.101544:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350792ca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+46:2388:x1870153002341248:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 244us (404us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.101554:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35626 00000100:00100000:3.0:1783520931.101576:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35628 00000100:00100000:3.0:1783520931.101600:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508059180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+45:2387:x1870153002341376:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.101611:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.101620:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 0000000042ef1cd5/0xd5401cbabc0223e3 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8ea:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cd9 expref: 44 pid: 8409 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.101639:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 0000000042ef1cd5/0xd5401cbabc0223e3 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8ea:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cd9 expref: 43 pid: 8409 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.101657:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000042ef1cd5/0xd5401cbabc0223e3 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x8ea:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cd9 expref: 43 pid: 8409 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.101674:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8ea:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.101681:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8ea:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.101692:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000855d7374/0xd5401cbabc024b7b lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.101722:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000855d7374/0xd5401cbabc024b7b lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x8ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.101858:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.101897:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508059180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+42:2387:x1870153002341376:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 312us (670us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.101930:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35628 00010000:00010000:3.0:1783520931.101965:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc024bac lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.101998:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc024bac lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.102102:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc024bac lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.102179:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 00000000381573cd/0xd5401cbabc024bac lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.102205:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to regular blp list, 0 blwis in pool 00000100:00100000:3.0:1783520931.102285:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507acd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+989:3626:x1870152993038848:12345-0@lo:6:osp-syn-0-0.0 Request processed in 5693us (5871us total) trans 4294977513 rc 0/0 00000100:00100000:3.0:1783520931.102299:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44029 00000100:00100000:3.0:1783520931.102326:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993040384 00000100:00100000:3.0:1783520931.102332:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993040384 00000100:00100000:3.0:1783520931.102399:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993040512 00000100:00100000:3.0:1783520931.102403:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993040512 00000100:00100000:3.0:1783520931.102429:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44035 00000100:00100000:3.0:1783520931.102436:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121b0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+990:3627:x1870152993040384:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.102492:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2294 count 1 00010000:00010000:3.0:1783520931.102505:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024bcf lrc: 3/0,1 mode: --/PW res: [0x280000401:0x8f6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.102522:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000f62dd5cc/0xd5401cbabc022437 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x8f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d81 expref: 41 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.102529:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8f6:0x0].0x0 since=33s v=(0 0) 00010000:00010000:3.0:1783520931.102533:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8f6:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.102537:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024bcf lrc: 4/0,1 mode: --/PW res: [0x280000401:0x8f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.102558:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000f62dd5cc/0xd5401cbabc022437 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d81 expref: 42 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.102570:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000f62dd5cc/0xd5401cbabc022437 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d81 expref: 42 pid: 20140 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.102582:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9ce00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0000_UUID:12734:1870152993040640:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.102640:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350a20d540 going to sleep for 11 seconds 00000004:00080000:3.0:1783520931.102727:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507938700/2, rc 0, transno 4294977510 00000100:00100000:3.0:1783520931.102738:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993038720:0@lo:6:osp-syn-0-0.0 00000004:00080000:3.0:1783520931.102751:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515303480/2, rc 0, transno 4294977513 00000100:00100000:3.0:1783520931.102817:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515303480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993038848:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.102940:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993040896:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.102974:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.103000:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1783520931.103009:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993040768:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.103106:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:3.0:1783520931.103489:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e451c00/2, rc 0, transno 4294977514 00000100:00100000:3.0:1783520931.103505:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e451c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993039872:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.103607:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9ce00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0000_UUID:12734:1870152993040640:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.103636:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024bcf lrc: 3/0,1 mode: --/PW res: [0x280000401:0x8f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.103782:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993041152 00000100:00100000:3.0:1783520931.103787:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993041152 00000100:00100000:3.0:1783520931.103815:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44039 00000100:00100000:3.0:1783520931.103824:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d99500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+993:3627:x1870152993041152:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.103839:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2298 count 1 00010000:00010000:3.0:1783520931.103851:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024be4 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x8fa:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.103872:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 0000000054ca4902/0xd5401cbabc02244c lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x8fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2db9 expref: 38 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.103888:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8fa:0x0].0x0 since=33s v=(0 0) 00010000:00010000:3.0:1783520931.103894:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8fa:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.103904:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024be4 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x8fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.103935:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 0000000054ca4902/0xd5401cbabc02244c lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2db9 expref: 38 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.103946:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 0000000054ca4902/0xd5401cbabc02244c lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2db9 expref: 38 pid: 9151 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.103959:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9ce00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0000_UUID:178496:1870152993041536:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.103989:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350a20d540 going to sleep for 11 seconds 00000100:00100000:3.0:1783520931.104071:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520931.104327:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1783520931.104341:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993041664:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.104404:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520931.104570:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024bcf lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.104579:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024bcf lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.104609:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024bcf lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.104622:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024bcf lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.104641:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520931.104739:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121b0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+996:3627:x1870152993040384:12345-0@lo:6:osp-syn-0-0.0 Request processed in 2302us (2600us total) trans 4294977517 rc 0/0 00000100:00100000:3.0:1783520931.104866:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44035 00000100:00100000:3.0:1783520931.104974:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993042176:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.105013:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:3.0:1783520931.105143:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9f800/2, rc 0, transno 4294977518 00000100:00100000:3.0:1783520931.105154:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993040128:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.105239:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9ce00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0000_UUID:178496:1870152993041536:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.105258:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024be4 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x8fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.105457:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002342144 00000100:00100000:3.0:1783520931.105463:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002342144 00010000:00010000:3.0:1783520931.105482:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000915c97f3/0xd5401cbabc02243e lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d9d expref: 37 pid: 12734 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.105504:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35634 00000100:00100000:3.0:1783520931.105519:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121b2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+38:2388:x1870153002342144:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.105599:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.105606:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000915c97f3/0xd5401cbabc02243e lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8f8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d9d expref: 38 pid: 12734 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.105619:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000915c97f3/0xd5401cbabc02243e lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8f8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d9d expref: 38 pid: 12734 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.105639:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000915c97f3/0xd5401cbabc02243e lrc: 0/0,0 mode: --/PR res: [0x280000401:0x8f8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d9d expref: 37 pid: 12734 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.105651:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8f8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.105654:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8f8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.105658:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000dc1a697e/0xd5401cbabc024bdd lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.105673:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000dc1a697e/0xd5401cbabc024bdd lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x8f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.105689:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002342400 00000100:00100000:3.0:1783520931.105692:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002342400 00010000:00010000:3.0:1783520931.105713:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000003680c3ad/0xd5401cbabc022445 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2dab expref: 37 pid: 20140 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.105909:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002342656 00000100:00100000:3.0:1783520931.105913:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002342656 00010000:00010000:3.0:1783520931.106019:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000a75a4212/0xd5401cbabc022461 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2dd5 expref: 35 pid: 8410 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.106040:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002342784 00000100:00100000:3.0:1783520931.106047:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002342784 00010000:00010000:3.0:1783520931.106070:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000000fa48b86/0xd5401cbabc022430 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x8f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d8f expref: 37 pid: 12734 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.106102:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35638 00000100:00100000:3.0:1783520931.106162:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121b3800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+37:2387:x1870153002342528:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.106235:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.106246:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 0000000084e09e8b/0xd5401cbabc02245a lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x8fb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2dc7 expref: 36 pid: 35274 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.106264:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 0000000084e09e8b/0xd5401cbabc02245a lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8fb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2dc7 expref: 35 pid: 35274 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.106284:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8fb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.106292:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8fb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.106347:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000c721a958/0xd5401cbabc024beb lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8fb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.106375:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000c721a958/0xd5401cbabc024beb lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x8fb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.106438:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.106470:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121b3800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+34:2387:x1870153002342528:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 351us (478us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.106497:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35638 00000100:00100000:3.0:1783520931.106544:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002342912 00000100:00100000:3.0:1783520931.106551:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002342912 00010000:00010000:3.0:1783520931.106571:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 0000000018827358/0xd5401cbabc022468 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2de3 expref: 34 pid: 20140 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.106665:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35641 00000100:00100000:3.0:1783520931.106683:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435130f6680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+34:2387:x1870153002342912:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.106693:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.106699:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 0000000018827358/0xd5401cbabc022468 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8fd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2de3 expref: 34 pid: 20140 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.106746:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 0000000018827358/0xd5401cbabc022468 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8fd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2de3 expref: 34 pid: 20140 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.106870:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 0000000018827358/0xd5401cbabc022468 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x8fd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2de3 expref: 34 pid: 20140 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.106911:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8fd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.106920:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8fd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.106936:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc024c00 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.106987:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000d023bc59/0xd5401cbabc024c00 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x8fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.107021:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.107057:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435130f6680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+32:2387:x1870153002342912:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 374us (584us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.107125:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35641 00010000:00010000:3.0:1783520931.107151:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024be4 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.107172:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024be4 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.107184:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024be4 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.107195:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024be4 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.107215:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000004:00080000:3.0:1783520931.107279:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9ed80/2, rc 0, transno 4294977519 00000100:00100000:3.0:1783520931.107297:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993041664:0@lo:6:osp-syn-0-0.0 00000004:00080000:3.0:1783520931.107321:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9ca80/2, rc 0, transno 4294977520 00000100:00100000:3.0:1783520931.107335:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993040896:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.107728:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d99500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+1000:3627:x1870152993041152:12345-0@lo:6:osp-syn-0-0.0 Request processed in 3902us (4020us total) trans 4294977523 rc 0/0 00000100:00100000:3.0:1783520931.107755:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44039 00000100:00100000:3.0:1783520931.107797:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993042688 00000100:00100000:3.0:1783520931.107804:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993042688 00000100:00100000:3.0:1783520931.107856:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44044 00000100:00100000:3.0:1783520931.107865:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ae5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+999:3628:x1870152993042688:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.107882:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2303 count 1 00010000:00010000:3.0:1783520931.107905:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024c0e lrc: 3/0,1 mode: --/PW res: [0x280000401:0x8ff:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.107920:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000a133e23d/0xd5401cbabc022476 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x8ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2dff expref: 29 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.107928:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8ff:0x0].0x0 since=33s v=(0 0) 00010000:00010000:3.0:1783520931.107937:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x8ff:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.107957:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024c0e lrc: 4/0,1 mode: --/PW res: [0x280000401:0x8ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.107990:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000a133e23d/0xd5401cbabc022476 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x8ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2dff expref: 29 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.108006:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000a133e23d/0xd5401cbabc022476 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x8ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2dff expref: 29 pid: 20415 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.108018:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9ea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0000_UUID:178496:1870152993042944:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.108072:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350d1ce000 going to sleep for 11 seconds 00000100:00100000:3.0:1783520931.108165:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993043072:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.108240:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:3.0:1783520931.108282:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9dc00/2, rc 0, transno 4294977525 00000100:00100000:3.0:1783520931.108293:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993042176:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.108312:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1783520931.108319:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993042816:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.108473:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.108497:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993043456:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.108548:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.108629:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993043712:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.108672:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.108702:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993043712 00000100:00100000:3.0:1783520931.108708:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993043712 00000100:00100000:3.0:1783520931.108745:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44049 00000100:00100000:3.0:1783520931.108760:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+1004:3626:x1870152993043456:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.108789:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2308 count 1 00010000:00010000:3.0:1783520931.108807:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc024c31 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x904:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.108828:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 000000007c863259/0xd5401cbabc022499 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x904:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e45 expref: 29 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.108844:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x904:0x0].0x0 since=33s v=(0 0) 00010000:00010000:3.0:1783520931.108848:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x904:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.108853:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc024c31 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x904:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.108885:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 000000007c863259/0xd5401cbabc022499 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x904:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e45 expref: 29 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.108941:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 000000007c863259/0xd5401cbabc022499 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x904:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e45 expref: 29 pid: 35282 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.108956:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad0a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0000_UUID:12734:1870152993044224:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.108981:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350a20d540 going to sleep for 11 seconds 00010000:00010000:3.0:1783520931.109023:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.109046:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121b2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+29:2388:x1870153002342144:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 3532us (3611us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.109054:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35634 00000004:00080000:3.0:1783520931.109214:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9f480/2, rc 0, transno 4294977526 00000100:00100000:3.0:1783520931.109226:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993040768:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.109506:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9ea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0000_UUID:178496:1870152993042944:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.109533:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024c0e lrc: 3/0,1 mode: --/PW res: [0x280000401:0x8ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.109831:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024c0e lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.109840:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024c0e lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.109849:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024c0e lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x8ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.109856:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000003107de87/0xd5401cbabc024c0e lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x8ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.111120:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002343296 00000100:00100000:3.0:1783520931.111125:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002343296 00010000:00010000:3.0:1783520931.111148:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000005820722d/0xd5401cbabc02248b lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x903:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e37 expref: 29 pid: 8410 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.111175:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002343552 00000100:00100000:3.0:1783520931.111180:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002343552 00010000:00010000:3.0:1783520931.111202:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 0000000021ad1eff/0xd5401cbabc022484 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x901:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e1b expref: 31 pid: 9151 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.111230:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002343808 00000100:00100000:3.0:1783520931.111237:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002343808 00010000:00010000:3.0:1783520931.111258:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000007c863259/0xd5401cbabc022499 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x904:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e45 expref: 32 pid: 35282 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.111302:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35644 00000100:00100000:3.0:1783520931.111323:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94361b4a6850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+32:2388:x1870153002343424:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.111334:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.111341:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000039ad9f0/0xd5401cbabc022492 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x902:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e29 expref: 32 pid: 12734 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.111355:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000039ad9f0/0xd5401cbabc022492 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x902:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e29 expref: 32 pid: 12734 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.111405:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000039ad9f0/0xd5401cbabc022492 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x902:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e29 expref: 32 pid: 12734 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.111428:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x902:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.111433:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x902:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.111438:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 0000000000a37bb7/0xd5401cbabc024c15 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x902:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.111448:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000000a37bb7/0xd5401cbabc024c15 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x902:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.111471:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.111499:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94361b4a6850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+29:2388:x1870153002343424:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 184us (412us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.111510:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35644 00000100:00100000:3.0:1783520931.111567:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35646 00000100:00100000:3.0:1783520931.111582:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ae4380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+30:2388:x1870153002343680:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.111609:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.111618:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 0000000012f7f3a9/0xd5401cbabc02247d lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x900:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e0d expref: 30 pid: 35276 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.111656:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 0000000012f7f3a9/0xd5401cbabc02247d lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x900:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e0d expref: 29 pid: 35276 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.111678:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x900:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.111690:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x900:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.111696:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc024c2a lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x900:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.111719:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000354fa27c/0xd5401cbabc024c2a lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x900:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.111742:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.111897:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ae4380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+28:2388:x1870153002343680:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 315us (752us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.111944:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35646 00000100:00100000:3.0:1783520931.111962:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35649 00000100:00100000:3.0:1783520931.111973:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ae6a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+27:2390:x1870153002344064:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.111984:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.111989:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000d23a727e/0xd5401cbabc0224a0 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x906:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e61 expref: 27 pid: 8408 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.112006:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000d23a727e/0xd5401cbabc0224a0 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x906:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e61 expref: 27 pid: 8408 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.112032:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x906:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.112035:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x906:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.112039:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 0000000018903c53/0xd5401cbabc024c3f lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x906:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.112047:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000018903c53/0xd5401cbabc024c3f lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x906:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.112065:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.112084:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ae6a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+26:2390:x1870153002344064:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 112us (757us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.112114:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35649 00000100:00100000:3.0:1783520931.112203:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993044992:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.112247:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:3.0:1783520931.112304:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e1c6300/2, rc 0, transno 4294977529 00000100:00100000:3.0:1783520931.112320:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993043072:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.112448:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1783520931.112462:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993045120:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.112527:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:3.0:1783520931.112611:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9ce00/2, rc 0, transno 4294977531 00000100:00100000:3.0:1783520931.112623:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993042816:0@lo:6:osp-syn-0-0.0 00010000:00010000:3.0:1783520931.112683:0:12734:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000007c863259/0xd5401cbabc022499 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x904:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e45 expref: 23 pid: 35282 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.112708:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad0a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0000_UUID:12734:1870152993044224:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.112730:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc024c31 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x904:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.112761:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc024c31 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x904:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.112799:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc024c31 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x904:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.112852:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc024c31 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x904:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.112902:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 0000000036351fe2/0xd5401cbabc024c31 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x904:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.112922:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520931.113015:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+1009:3626:x1870152993043456:12345-0@lo:6:osp-syn-0-0.0 Request processed in 4257us (4472us total) trans 4294977532 rc 0/0 00000100:00100000:3.0:1783520931.113030:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44049 00000004:00080000:3.0:1783520931.113069:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e1c5880/2, rc 0, transno 4294977532 00000100:00100000:3.0:1783520931.113084:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993043456:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.113185:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1783520931.113198:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993045760:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.113266:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.113321:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993045760 00000100:00100000:3.0:1783520931.113328:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993045760 00000004:00080000:3.0:1783520931.113920:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ad2d80/2, rc 0, transno 4294977533 00000100:00100000:3.0:1783520931.113930:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993043712:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.114038:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1783520931.114049:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993046272:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.114083:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.114126:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993046272 00000100:00100000:3.0:1783520931.114134:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993046272 00000100:00100000:3.0:1783520931.114332:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993046400:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.114452:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.114884:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002344320 00000100:00100000:3.0:1783520931.114891:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002344320 00010000:00010000:3.0:1783520931.114915:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000f75254d9/0xd5401cbabc0224bc lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x909:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e8b expref: 20 pid: 35282 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.115054:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002344576 00000100:00100000:3.0:1783520931.115059:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002344576 00010000:00010000:3.0:1783520931.115079:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000007c9130df/0xd5401cbabc0224ca lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x90b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ea7 expref: 23 pid: 35275 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.115121:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002344832 00000100:00100000:3.0:1783520931.115130:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002344832 00010000:00010000:3.0:1783520931.115151:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000bf576d5a/0xd5401cbabc0224d1 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x90c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2eb5 expref: 24 pid: 20415 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.115172:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35653 00000100:00100000:3.0:1783520931.115179:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ae7100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+24:2388:x1870153002344576:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.115186:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.115193:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000007c9130df/0xd5401cbabc0224ca lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x90b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ea7 expref: 24 pid: 35275 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.115215:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000007c9130df/0xd5401cbabc0224ca lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x90b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ea7 expref: 24 pid: 35275 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.115296:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000007c9130df/0xd5401cbabc0224ca lrc: 0/0,0 mode: --/PR res: [0x280000401:0x90b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ea7 expref: 22 pid: 35275 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.115321:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x90b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.115324:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x90b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.115329:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc024c62 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x90b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.115342:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000008faaf9b7/0xd5401cbabc024c62 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x90b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.115521:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.115548:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ae7100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+17:2388:x1870153002344576:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 365us (555us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.115560:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35653 00000004:00080000:3.0:1783520931.115839:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ad1180/2, rc 0, transno 4294977536 00000100:00100000:3.0:1783520931.115854:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993045376:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.115909:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435130f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993046912:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.115954:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:3.0:1783520931.115986:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ad0e00/2, rc 0, transno 4294977538 00000100:00100000:3.0:1783520931.116001:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993044992:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.116051:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993047168:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.116100:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:3.0:1783520931.116130:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ad1880/2, rc 0, transno 4294977537 00000100:00100000:3.0:1783520931.116149:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993045760:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.116212:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993046912 00000100:00100000:3.0:1783520931.116215:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993046912 00000100:00100000:3.0:1783520931.116247:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993047168 00000100:00100000:3.0:1783520931.116251:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993047168 00000100:00100000:3.0:1783520931.116292:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44061 00000100:00100000:3.0:1783520931.116308:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+1016:3625:x1870152993046912:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.116325:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2320 count 1 00010000:00010000:3.0:1783520931.116341:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024c85 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x910:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.116411:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 00000000d71ed93a/0xd5401cbabc0224ed lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x910:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2eed expref: 14 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.116419:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x910:0x0].0x0 since=33s v=(0 0) 00010000:00010000:3.0:1783520931.116423:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x910:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.116427:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024c85 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x910:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.116459:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 00000000d71ed93a/0xd5401cbabc0224ed lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x910:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2eed expref: 14 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.116482:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 00000000d71ed93a/0xd5401cbabc0224ed lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x910:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2eed expref: 14 pid: 35276 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.116496:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0000_UUID:12734:1870152993047424:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.116543:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350d1ce000 going to sleep for 11 seconds 00000100:00100000:3.0:1783520931.116564:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002344960 00000100:00100000:3.0:1783520931.116569:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002344960 00010000:00010000:3.0:1783520931.116611:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000c18599ac/0xd5401cbabc0224d8 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x90d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ec3 expref: 15 pid: 9151 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.116649:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002345088 00000100:00100000:3.0:1783520931.116702:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002345088 00010000:00010000:3.0:1783520931.116726:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000000eae7ecf/0xd5401cbabc0224df lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x90e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ed1 expref: 16 pid: 8409 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.116754:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35656 00000100:00100000:3.0:1783520931.116772:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e47b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+16:2390:x1870153002344960:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.116799:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.116810:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000c18599ac/0xd5401cbabc0224d8 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x90d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ec3 expref: 16 pid: 9151 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.116900:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000c18599ac/0xd5401cbabc0224d8 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x90d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ec3 expref: 16 pid: 9151 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.116969:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x90d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.116973:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x90d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.116978:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc024c70 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x90d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.116987:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000efd5ffe8/0xd5401cbabc024c70 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x90d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.117015:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.117041:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e47b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+15:2390:x1870153002344960:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 271us (585us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.117055:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35656 00000100:00100000:3.0:1783520931.117414:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993047808 00000100:00100000:3.0:1783520931.117421:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993047808 00000100:00100000:3.0:1783520931.117453:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44064 00000100:00100000:3.0:1783520931.117465:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ace300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+1020:3628:x1870152993047808:12345-0@lo:6:osp-syn-0-0.0 00002000:00080000:3.0:1783520931.117479:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0000: Destroy object 0x280000401:2323 count 1 00010000:00010000:3.0:1783520931.117496:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024c9a lrc: 3/0,1 mode: --/PW res: [0x280000401:0x913:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.117512:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 0000000046547fa7/0xd5401cbabc022502 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x913:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f17 expref: 12 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.117548:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x913:0x0].0x0 since=33s v=(0 0) 00010000:00010000:3.0:1783520931.117555:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x913:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.117562:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024c9a lrc: 4/0,1 mode: --/PW res: [0x280000401:0x913:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.117600:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 0000000046547fa7/0xd5401cbabc022502 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x913:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f17 expref: 12 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.117640:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 0000000046547fa7/0xd5401cbabc022502 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x913:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f17 expref: 12 pid: 35282 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.117696:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0000_UUID:35275:1870152993047936:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520931.117835:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350a20d540 going to sleep for 11 seconds 00000100:00100000:3.0:1783520931.118017:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0000_UUID:12734:1870152993047424:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.118048:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024c85 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x910:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.118758:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002345344 00000100:00100000:3.0:1783520931.118764:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002345344 00010000:00010000:3.0:1783520931.118785:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000d71ed93a/0xd5401cbabc0224ed lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x910:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2eed expref: 12 pid: 35276 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.118818:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35659 00000100:00100000:3.0:1783520931.118827:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ae4a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+12:2388:x1870153002345344:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.118836:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.118841:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000d71ed93a/0xd5401cbabc0224ed lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x910:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2eed expref: 12 pid: 35276 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.118859:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000d71ed93a/0xd5401cbabc0224ed lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x910:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2eed expref: 12 pid: 35276 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.118924:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000d71ed93a/0xd5401cbabc0224ed lrc: 0/0,0 mode: --/PR res: [0x280000401:0x910:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2eed expref: 12 pid: 35276 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.118940:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x910:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.118944:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x910:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.118949:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024c85 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x910:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.118960:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024c85 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x910:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.118992:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.119015:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ae4a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+11:2388:x1870153002345344:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 190us (295us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.119022:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35659 00010000:00010000:3.0:1783520931.119038:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024c85 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x910:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.119046:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024c85 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x910:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.119055:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024c85 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x910:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.119062:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 0000000058a3983d/0xd5401cbabc024c85 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x910:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.119079:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520931.119202:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+1022:3625:x1870152993046912:12345-0@lo:6:osp-syn-0-0.0 Request processed in 2898us (3253us total) trans 4294977545 rc 0/0 00000100:00100000:3.0:1783520931.119210:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44061 00000100:00100000:3.0:1783520931.119248:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0000_UUID:35275:1870152993047936:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520931.119265:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 00000000b97c0e36/0xd5401cbabc024c9a lrc: 3/0,1 mode: --/PW res: [0x280000401:0x913:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.119487:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002345600 00000100:00100000:3.0:1783520931.119493:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002345600 00010000:00010000:3.0:1783520931.119565:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000003a46f0c9/0xd5401cbabc0224fb lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x912:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f09 expref: 12 pid: 20415 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.119584:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35660 00000100:00100000:3.0:1783520931.119604:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507939880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+13:2388:x1870153002345472:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.119746:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.119753:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000002d185524/0xd5401cbabc0224f4 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x911:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2efb expref: 13 pid: 12734 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.119771:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000002d185524/0xd5401cbabc0224f4 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x911:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2efb expref: 12 pid: 12734 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.119799:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000002d185524/0xd5401cbabc0224f4 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x911:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2efb expref: 11 pid: 12734 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.119823:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x911:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.119829:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x911:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520931.119838:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000005f11335d/0xd5401cbabc024c8c lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x911:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.119848:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000005f11335d/0xd5401cbabc024c8c lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x911:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.119870:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.119896:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507939880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+9:2388:x1870153002345472:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 292us (405us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.119910:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35660 00000004:00080000:3.0:1783520931.120040:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9c700/2, rc 0, transno 4294977546 00000100:00100000:3.0:1783520931.120050:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993047168:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520931.120158:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002345856 00000100:00100000:3.0:1783520931.120166:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002345856 00010000:00010000:3.0:1783520931.120195:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000009a2950ad/0xd5401cbabc022509 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x914:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f25 expref: 8 pid: 8409 timeout: 4226 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.120417:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002345984 00000100:00100000:3.0:1783520931.120424:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002345984 00010000:00010000:3.0:1783520931.120448:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 00000000ed3e104a/0xd5401cbabc022bca lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x915:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777211904->16777215999) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f32b3 expref: 8 pid: 9151 timeout: 4226 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520931.120522:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35664 00000100:00100000:3.0:1783520931.120535:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ae4000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+7:2390:x1870153002345984:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520931.120546:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520931.120601:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 00000000ed3e104a/0xd5401cbabc022bca lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x915:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777211904->16777215999) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f32b3 expref: 7 pid: 9151 timeout: 4226 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.120621:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 00000000ed3e104a/0xd5401cbabc022bca lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x915:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777211904->16777215999) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f32b3 expref: 7 pid: 9151 timeout: 4226 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.120637:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 00000000ed3e104a/0xd5401cbabc022bca lrc: 0/0,0 mode: --/PW res: [0x280000401:0x915:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777211904->16777215999) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f32b3 expref: 7 pid: 9151 timeout: 4226 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.120651:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x915:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520931.120654:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x915:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520931.120659:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc024ca8 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x915:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.120668:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc024ca8 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x915:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520931.120701:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520931.120738:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ae4000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+6:2390:x1870153002345984:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 207us (415us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.120757:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35664 00000100:00100000:3.0:1783520931.253704:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507acf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3627:1870152993049216:0@lo:400:kworker.0 00000100:00100000:3.0:1783520931.253777:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.253859:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507acc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3627:1870152993049728:0@lo:400:kworker.0 00000100:00100000:3.0:1783520931.253919:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.253992:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ace300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993050368:0@lo:400:kworker.0 00000100:00100000:3.0:1783520931.254043:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.254088:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:3.0:1783520931.254111:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507acc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3627:1870152993049856:0@lo:400:kworker.0 00000100:00100000:3.0:1783520931.254167:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.254216:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3627:1870152993050112:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1783520931.254278:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.254352:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993049600 00000100:00100000:3.0:1783520931.254413:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993049600 00000100:00100000:3.0:1783520931.254474:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993050368 00000100:00100000:3.0:1783520931.254480:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993050368 00000100:00100000:3.0:1783520931.254524:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4753 00000100:00100000:3.0:1783520931.254548:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507acf800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0001-mdtlov_UUID+5:3626:x1870152993049600:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1783520931.254640:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507acf800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0001-mdtlov_UUID+5:3626:x1870152993049600:12345-0@lo:400:kworker.0 Request processed in 100us (680us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.254658:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4753 00000100:00100000:3.0:1783520931.254690:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993049984 00000100:00100000:3.0:1783520931.254697:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993049984 00000100:00100000:3.0:1783520931.254753:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4754 00000100:00100000:3.0:1783520931.254773:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e99c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+6:3627:x1870152993050368:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1783520931.254829:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e99c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+6:3627:x1870152993050368:12345-0@lo:400:kworker.0 Request processed in 63us (786us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.254849:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4754 00000100:00100000:3.0:1783520931.254890:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4755 00000100:00100000:3.0:1783520931.254910:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0001-mdtlov_UUID+5:3626:x1870152993049984:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1783520931.254974:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0001-mdtlov_UUID+5:3626:x1870152993049984:12345-0@lo:400:kworker.0 Request processed in 73us (417us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520931.254997:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4755 00000100:00100000:3.0:1783520931.255093:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520931.255149:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507acca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:ea7ffe8e-f913-4c2c-b57a-a81a86292f4f:3628:1870152993048960:0@lo:400:kworker.0 00000100:00100000:3.0:1783520931.255224:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507acc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3627:1870152993049856:0@lo:400:kworker.0 00000100:00100000:3.0:1783520931.255261:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3627:1870152993050112:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1783520931.255289:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507acf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3627:1870152993049216:0@lo:400:kworker.0 00000100:00100000:3.0:1783520931.255303:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507acc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3627:1870152993049728:0@lo:400:kworker.0 00000100:00100000:3.0:1783520931.255338:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ace300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993050368:0@lo:400:kworker.0 00000100:00100000:3.0:1783520931.255469:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507acdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0000_UUID:3628:1870152993049344:0@lo:400:kworker.0 00000100:00100000:3.0:1783520933.088528:0:35278:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002346624 00000100:00100000:3.0:1783520933.088537:0:35278:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002346624 00000100:00100000:3.0:1783520933.088606:0:35278:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98343 00000100:00100000:3.0:1783520933.088618:0:35278:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350793a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_009:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+14:251157:x1870153002346624:12345-192.168.204.33@tcp:101:touch.0 00010000:00010000:3.0:1783520933.088644:0:35278:0:(ldlm_lockd.c:1265:ldlm_handle_enqueue()) ### server-side enqueue handler START 00010000:00010000:3.0:1783520933.088668:0:35278:0:(ldlm_lockd.c:1343:ldlm_handle_enqueue()) ### server-side enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024cc4 lrc: 2/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT gid 0 flags: 0x40000000000000 nid: local remote: 0xc275876b859f341f expref: -99 pid: 35278 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.088760:0:35278:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784cc00/0xd5401cbabc024ccb lrc: 3/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x40000000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.088774:0:35278:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94350784cc00/0xd5401cbabc024ccb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT flags: 0x50210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.088794:0:35278:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94350784cc00/0xd5401cbabc024ccb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT flags: 0x40210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00000004:00200000:3.0:1783520933.088847:0:35278:0:(mdt_handler.c:909:mdt_pack_attr2body()) [0x200000007:0x1:0x0]: returning size 69632 00000004:00200000:3.0:1783520933.088851:0:35278:0:(mdt_handler.c:1574:mdt_getattr_internal()) dirent count 0 stripe count 1 MDT count 2 00000004:00010000:3.0:1783520933.088871:0:35278:0:(mdt_handler.c:2507:mdt_getattr_name_lock()) ### Returning lock to client ns: mdt-lustre-MDT0000_UUID lock: ffff94350784cc00/0xd5401cbabc024ccb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT flags: 0x40210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.088896:0:35278:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024cc4 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x44000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f341f expref: 16 pid: 35278 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.088914:0:35278:0:(ldlm_lockd.c:1513:ldlm_handle_enqueue()) ### server-side enqueue handler, sending reply (err=0, rc=0) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784cc00/0xd5401cbabc024ccb lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x40200000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f341f expref: 15 pid: 35278 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.088922:0:35278:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x0 (ffff943504b72c00) 00010000:00010000:3.0:1783520933.088925:0:35278:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 13 00010000:00010000:3.0:1783520933.088929:0:35278:0:(ldlm_lockd.c:1592:ldlm_handle_enqueue()) ### server-side enqueue handler END (lock ffff94350784cc00, rc 0) 00000100:00100000:3.0:1783520933.088984:0:35278:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350793a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_009:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+15:251157:x1870153002346624:12345-192.168.204.33@tcp:101:touch.0 Request processed in 369us (513us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520933.088991:0:35278:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98343 00000100:00100000:3.0:1783520933.090042:0:6494:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002346752 00000100:00100000:3.0:1783520933.090049:0:6494:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002346752 00000100:00100000:3.0:1783520933.090088:0:6494:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98344 00000100:00100000:3.0:1783520933.090097:0:6494:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350793b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+15:251157:x1870153002346752:12345-192.168.204.33@tcp:101:touch.0 00010000:00010000:3.0:1783520933.090109:0:6494:0:(ldlm_lockd.c:1265:ldlm_handle_enqueue()) ### server-side enqueue handler START 00010000:00010000:3.0:1783520933.090120:0:6494:0:(ldlm_lockd.c:1343:ldlm_handle_enqueue()) ### server-side enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024cd2 lrc: 2/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT gid 0 flags: 0x40000000000000 nid: local remote: 0xc275876b859f3426 expref: -99 pid: 6494 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090176:0:6494:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784d000/0xd5401cbabc024cd9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x40000000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090184:0:6494:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94350784d000/0xd5401cbabc024cd9 lrc: 3/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090190:0:6494:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94350784d000/0xd5401cbabc024cd9 lrc: 3/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090198:0:6494:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ce00/0xd5401cbabc024ce0 lrc: 3/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x121d381d bits 0x0/0x0 rrc: 3 type: IBT flags: 0x40000000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090202:0:6494:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ce00/0xd5401cbabc024ce0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x121d381d bits 0x2/0x0 rrc: 3 type: IBT flags: 0x50210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090208:0:6494:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ce00/0xd5401cbabc024ce0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x121d381d bits 0x2/0x0 rrc: 3 type: IBT flags: 0x40210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090293:0:6494:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c800/0xd5401cbabc024ce7 lrc: 3/1,0 mode: --/PR res: [0x200000402:0x726:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x40000000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090300:0:6494:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c800/0xd5401cbabc024ce7 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x726:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT flags: 0x50210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090306:0:6494:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c800/0xd5401cbabc024ce7 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x726:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT flags: 0x40210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00000004:00200000:3.0:1783520933.090330:0:6494:0:(mdt_handler.c:909:mdt_pack_attr2body()) [0x200000402:0x726:0x0]: returning size 4096 00000004:00200000:3.0:1783520933.090332:0:6494:0:(mdt_handler.c:1574:mdt_getattr_internal()) dirent count 0 stripe count 1 MDT count 2 00000004:00010000:3.0:1783520933.090341:0:6494:0:(mdt_handler.c:2507:mdt_getattr_name_lock()) ### Returning lock to client ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c800/0xd5401cbabc024ce7 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x726:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT flags: 0x40210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090350:0:6494:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784d000/0xd5401cbabc024cd9 lrc: 4/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090354:0:6494:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784d000/0xd5401cbabc024cd9 lrc: 4/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090399:0:6494:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff94350784d000/0xd5401cbabc024cd9 lrc: 3/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090404:0:6494:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff94350784d000/0xd5401cbabc024cd9 lrc: 4/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090408:0:6494:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784d000 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520933.090412:0:6494:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784d000/0xd5401cbabc024cd9 lrc: 4/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090417:0:6494:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784d000/0xd5401cbabc024cd9 lrc: 5/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40218400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090425:0:6494:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x0 (ffff943504b72c00) 00010000:00010000:3.0:1783520933.090428:0:6494:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520933.090430:0:6494:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff94350784d000/0xd5401cbabc024cd9 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x44a19400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090435:0:6494:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94350784d000/0xd5401cbabc024cd9 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x44a19400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090446:0:6494:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ce00/0xd5401cbabc024ce0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x121d381d bits 0x2/0x0 rrc: 3 type: IBT flags: 0x50210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090450:0:6494:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ce00/0xd5401cbabc024ce0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x121d381d bits 0x2/0x0 rrc: 3 type: IBT flags: 0x50210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090454:0:6494:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ce00/0xd5401cbabc024ce0 lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x121d381d bits 0x2/0x0 rrc: 3 type: IBT flags: 0x40210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090456:0:6494:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784ce00 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520933.090458:0:6494:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ce00/0xd5401cbabc024ce0 lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x121d381d bits 0x2/0x0 rrc: 3 type: IBT flags: 0x40210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090462:0:6494:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ce00/0xd5401cbabc024ce0 lrc: 4/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x121d381d bits 0x2/0x0 rrc: 3 type: IBT flags: 0x40218400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090466:0:6494:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x121d381d (ffff9435163be500) 00010000:00010000:3.0:1783520933.090467:0:6494:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520933.090469:0:6494:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ce00/0xd5401cbabc024ce0 lrc: 2/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x121d381d bits 0x2/0x0 rrc: 3 type: IBT flags: 0x44a19400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090472:0:6494:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ce00/0xd5401cbabc024ce0 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x121d381d bits 0x2/0x0 rrc: 3 type: IBT flags: 0x44a19400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090489:0:6494:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024cd2 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x44000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3426 expref: 17 pid: 6494 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090498:0:6494:0:(ldlm_lockd.c:1513:ldlm_handle_enqueue()) ### server-side enqueue handler, sending reply (err=0, rc=0) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c800/0xd5401cbabc024ce7 lrc: 3/0,0 mode: PR/PR res: [0x200000402:0x726:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x40200000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3426 expref: 16 pid: 6494 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.090506:0:6494:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x726:0x0].0x0 (ffff9435163beb00) 00010000:00010000:3.0:1783520933.090507:0:6494:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 11 00010000:00010000:3.0:1783520933.090509:0:6494:0:(ldlm_lockd.c:1592:ldlm_handle_enqueue()) ### server-side enqueue handler END (lock ffff94350784c800, rc 0) 00000100:00100000:3.0:1783520933.090557:0:6494:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350793b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+16:251157:x1870153002346752:12345-192.168.204.33@tcp:101:touch.0 Request processed in 452us (561us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520933.090564:0:6494:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98344 00000100:00100000:3.0:1783520933.096631:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520933.098192:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002347520 00000100:00100000:3.0:1783520933.098203:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002347520 00000100:00100000:3.0:1783520933.098258:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 44068 00000100:00100000:3.0:1783520933.098269:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+988:2387:x1870153002347520:12345-192.168.204.33@tcp:2:touch.0 00000100:00100000:3.0:1783520933.098458:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+989:2387:x1870153002347520:12345-192.168.204.33@tcp:2:touch.0 Request processed in 192us (322us total) trans 4294975573 rc 0/0 00000100:00100000:3.0:1783520933.098463:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 44068 00000100:00100000:3.0:1783520933.099456:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520933.108171:0:35279:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002348032 00000100:00100000:3.0:1783520933.108182:0:35279:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002348032 00000100:00100000:3.0:1783520933.108243:0:35279:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98348 00000100:00100000:3.0:1783520933.108260:0:35279:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_010:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+22:251161:x1870153002348032:12345-192.168.204.33@tcp:101:lfs.0 00010000:00010000:3.0:1783520933.108283:0:35279:0:(ldlm_lockd.c:1265:ldlm_handle_enqueue()) ### server-side enqueue handler START 00010000:00010000:3.0:1783520933.108303:0:35279:0:(ldlm_lockd.c:1343:ldlm_handle_enqueue()) ### server-side enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024d1f lrc: 2/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT gid 0 flags: 0x40000000000000 nid: local remote: 0xc275876b859f343b expref: -99 pid: 35279 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00200000:3.0:1783520933.108446:0:35279:0:(mdt_handler.c:909:mdt_pack_attr2body()) [0x200000402:0x726:0x0]: returning size 4096 00010000:00010000:3.0:1783520933.108567:0:35279:0:(ldlm_lockd.c:1513:ldlm_handle_enqueue()) ### server-side enqueue handler, sending reply (err=301, rc=0) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024d1f lrc: 1/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x44000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f343b expref: 24 pid: 35279 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.108582:0:35279:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x0 (ffff943504b72c00) 00010000:00010000:3.0:1783520933.108585:0:35279:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 1 00010000:00010000:3.0:1783520933.108591:0:35279:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024d1f lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x44801000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f343b expref: 24 pid: 35279 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.108604:0:35279:0:(ldlm_lockd.c:1592:ldlm_handle_enqueue()) ### server-side enqueue handler END (lock ffff94350784c200, rc 0) 00000100:00100000:3.0:1783520933.108669:0:35279:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_010:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+23:251161:x1870153002348032:12345-192.168.204.33@tcp:101:lfs.0 Request processed in 417us (567us total) trans 8589957877 rc 301/301 00000100:00100000:3.0:1783520933.108678:0:35279:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98348 00000100:00100000:3.0:1783520933.109624:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520933.122076:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520933.143443:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520933.165485:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520933.172655:0:6494:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002350848 00000100:00100000:3.0:1783520933.172664:0:6494:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002350848 00000100:00100000:3.0:1783520933.172725:0:6494:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98358 00000100:00100000:3.0:1783520933.172736:0:6494:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d91500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+34:251173:x1870153002350848:12345-192.168.204.33@tcp:36:lfs.0 00010000:00010000:3.0:1783520933.172807:0:6494:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024db2 lrc: 3/0,1 mode: --/CW res: [0x200000402:0x726:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x40000000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.172820:0:6494:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024db2 lrc: 3/0,1 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.172828:0:6494:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024db2 lrc: 3/0,1 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.172842:0:6494:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024db9 lrc: 3/0,1 mode: --/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x0/0x0 rrc: 2 type: IBT flags: 0x40000000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.172850:0:6494:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024db9 lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.172856:0:6494:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024db9 lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.172961:0:6494:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024db2 lrc: 4/0,1 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.172966:0:6494:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024db2 lrc: 4/0,1 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.172970:0:6494:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024db2 lrc: 3/0,0 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.172975:0:6494:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024db2 lrc: 4/0,0 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.172978:0:6494:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784de00 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520933.172981:0:6494:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024db2 lrc: 4/0,0 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.172986:0:6494:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024db2 lrc: 5/0,0 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40218400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.172994:0:6494:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x726:0x0].0x0 (ffff9435163beb00) 00010000:00010000:3.0:1783520933.172996:0:6494:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520933.172999:0:6494:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024db2 lrc: 3/0,0 mode: --/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x44a19400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.173004:0:6494:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024db2 lrc: 0/0,0 mode: --/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x44a19400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.173015:0:6494:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024db9 lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.173019:0:6494:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024db9 lrc: 2/0,0 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.173022:0:6494:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024db9 lrc: 3/0,0 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.173025:0:6494:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784c200 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520933.173026:0:6494:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024db9 lrc: 3/0,0 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.173029:0:6494:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024db9 lrc: 4/0,0 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40218400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.173033:0:6494:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x726:0x0].0x151df48e (ffff943508e9e100) 00010000:00010000:3.0:1783520933.173034:0:6494:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520933.173035:0:6494:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024db9 lrc: 2/0,0 mode: --/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.173038:0:6494:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024db9 lrc: 0/0,0 mode: --/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520933.173101:0:6494:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d91500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+35:251173:x1870153002350848:12345-192.168.204.33@tcp:36:lfs.0 Request processed in 369us (510us total) trans 0 rc -2/-2 00000100:00100000:3.0:1783520933.173107:0:6494:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98358 00000100:00100000:3.0:1783520933.203875:0:35279:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002351872 00000100:00100000:3.0:1783520933.203884:0:35279:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002351872 00000100:00100000:3.0:1783520933.203933:0:35279:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98362 00000100:00100000:3.0:1783520933.203944:0:35279:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_010:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+38:251177:x1870153002351872:12345-192.168.204.33@tcp:36:lfs.0 00010000:00010000:3.0:1783520933.204032:0:35279:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024dea lrc: 3/0,1 mode: --/CW res: [0x200000402:0x726:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x40000000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.204049:0:35279:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024dea lrc: 3/0,1 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.204060:0:35279:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024dea lrc: 3/0,1 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.204072:0:35279:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024df1 lrc: 3/0,1 mode: --/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x0/0x0 rrc: 2 type: IBT flags: 0x40000000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.204079:0:35279:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024df1 lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.204086:0:35279:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024df1 lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.204199:0:35279:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024dea lrc: 4/0,1 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.204207:0:35279:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024dea lrc: 4/0,1 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.204212:0:35279:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024dea lrc: 3/0,0 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.204218:0:35279:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024dea lrc: 4/0,0 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.204223:0:35279:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784c200 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520933.204227:0:35279:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024dea lrc: 4/0,0 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.204234:0:35279:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024dea lrc: 5/0,0 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40218400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.204245:0:35279:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x726:0x0].0x0 (ffff9435163beb00) 00010000:00010000:3.0:1783520933.204248:0:35279:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520933.204256:0:35279:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024dea lrc: 3/0,0 mode: --/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x44a19400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.204263:0:35279:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024dea lrc: 0/0,0 mode: --/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x44a19400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.204278:0:35279:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024df1 lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.204282:0:35279:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024df1 lrc: 2/0,0 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.204286:0:35279:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024df1 lrc: 3/0,0 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.204289:0:35279:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784de00 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520933.204290:0:35279:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024df1 lrc: 3/0,0 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.204293:0:35279:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024df1 lrc: 4/0,0 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40218400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.204298:0:35279:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x726:0x0].0x151df48e (ffff943508e9e100) 00010000:00010000:3.0:1783520933.204299:0:35279:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520933.204301:0:35279:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024df1 lrc: 2/0,0 mode: --/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.204305:0:35279:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024df1 lrc: 0/0,0 mode: --/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520933.204416:0:35279:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_010:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+39:251177:x1870153002351872:12345-192.168.204.33@tcp:36:lfs.0 Request processed in 474us (591us total) trans 0 rc -2/-2 00000100:00100000:3.0:1783520933.204423:0:35279:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98362 00000100:00100000:3.0:1783520933.210470:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520933.285398:0:6494:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002354432 00000100:00100000:3.0:1783520933.285407:0:6494:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002354432 00000100:00100000:3.0:1783520933.285467:0:6494:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98372 00000100:00100000:3.0:1783520933.285479:0:6494:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d99f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+48:251187:x1870153002354432:12345-192.168.204.33@tcp:36:lfs.0 00010000:00010000:3.0:1783520933.285570:0:6494:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024e76 lrc: 3/0,1 mode: --/CW res: [0x200000402:0x726:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x40000000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.285586:0:6494:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024e76 lrc: 3/0,1 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.285598:0:6494:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024e76 lrc: 3/0,1 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.285613:0:6494:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024e7d lrc: 3/0,1 mode: --/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x0/0x0 rrc: 2 type: IBT flags: 0x40000000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.285620:0:6494:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024e7d lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.285627:0:6494:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024e7d lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.285744:0:6494:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024e76 lrc: 4/0,1 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.285753:0:6494:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024e76 lrc: 4/0,1 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.285758:0:6494:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024e76 lrc: 3/0,0 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.285764:0:6494:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024e76 lrc: 4/0,0 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.285770:0:6494:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784de00 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520933.285774:0:6494:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024e76 lrc: 4/0,0 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.285781:0:6494:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024e76 lrc: 5/0,0 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40218400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.285793:0:6494:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x726:0x0].0x0 (ffff9435163beb00) 00010000:00010000:3.0:1783520933.285796:0:6494:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520933.285801:0:6494:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024e76 lrc: 3/0,0 mode: --/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x44a19400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.285807:0:6494:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024e76 lrc: 0/0,0 mode: --/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x44a19400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.285822:0:6494:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024e7d lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.285829:0:6494:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024e7d lrc: 2/0,0 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.285835:0:6494:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024e7d lrc: 3/0,0 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.285839:0:6494:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784c200 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520933.285841:0:6494:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024e7d lrc: 3/0,0 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.285845:0:6494:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024e7d lrc: 4/0,0 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40218400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.285849:0:6494:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x726:0x0].0x151df48e (ffff943508e9e100) 00010000:00010000:3.0:1783520933.285851:0:6494:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520933.285852:0:6494:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024e7d lrc: 2/0,0 mode: --/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.285856:0:6494:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024e7d lrc: 0/0,0 mode: --/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520933.285930:0:6494:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d99f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+49:251187:x1870153002354432:12345-192.168.204.33@tcp:36:lfs.0 Request processed in 455us (626us total) trans 0 rc -2/-2 00000100:00100000:3.0:1783520933.285938:0:6494:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98372 00000100:00100000:3.0:1783520933.318667:0:35279:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002355456 00000100:00100000:3.0:1783520933.318682:0:35279:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002355456 00000100:00100000:3.0:1783520933.318755:0:35279:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98376 00000100:00100000:3.0:1783520933.318770:0:35279:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c56e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_010:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+52:251191:x1870153002355456:12345-192.168.204.33@tcp:36:lfs.0 00010000:00010000:3.0:1783520933.318879:0:35279:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024eae lrc: 3/0,1 mode: --/CW res: [0x200000402:0x726:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x40000000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.318898:0:35279:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024eae lrc: 3/0,1 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.318914:0:35279:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024eae lrc: 3/0,1 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.318931:0:35279:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024eb5 lrc: 3/0,1 mode: --/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x0/0x0 rrc: 2 type: IBT flags: 0x40000000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.318938:0:35279:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024eb5 lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.318945:0:35279:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024eb5 lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.319092:0:35279:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024eae lrc: 4/0,1 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.319098:0:35279:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024eae lrc: 4/0,1 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.319103:0:35279:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024eae lrc: 3/0,0 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.319107:0:35279:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024eae lrc: 4/0,0 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.319111:0:35279:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784c200 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520933.319114:0:35279:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024eae lrc: 4/0,0 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.319119:0:35279:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024eae lrc: 5/0,0 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40218400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.319129:0:35279:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x726:0x0].0x0 (ffff9435163beb00) 00010000:00010000:3.0:1783520933.319132:0:35279:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520933.319135:0:35279:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024eae lrc: 3/0,0 mode: --/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x44a19400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.319139:0:35279:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024eae lrc: 0/0,0 mode: --/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x44a19400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.319152:0:35279:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024eb5 lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.319156:0:35279:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024eb5 lrc: 2/0,0 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.319159:0:35279:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024eb5 lrc: 3/0,0 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.319162:0:35279:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784de00 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520933.319164:0:35279:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024eb5 lrc: 3/0,0 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.319167:0:35279:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024eb5 lrc: 4/0,0 mode: PW/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40218400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.319171:0:35279:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x726:0x0].0x151df48e (ffff943508e9e100) 00010000:00010000:3.0:1783520933.319172:0:35279:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520933.319174:0:35279:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024eb5 lrc: 2/0,0 mode: --/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.319177:0:35279:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024eb5 lrc: 0/0,0 mode: --/PW res: [0x200000402:0x726:0x0].0x151df48e bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520933.319283:0:35279:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c56e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_010:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+53:251191:x1870153002355456:12345-192.168.204.33@tcp:36:lfs.0 Request processed in 516us (690us total) trans 0 rc -2/-2 00000100:00100000:3.0:1783520933.319291:0:35279:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98376 00000100:00100000:3.0:1783520933.381552:0:6494:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002357888 00000100:00100000:3.0:1783520933.381559:0:6494:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002357888 00000100:00100000:3.0:1783520933.381605:0:6494:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98386 00000100:00100000:3.0:1783520933.381615:0:6494:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515f0a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+63:251198:x1870153002357888:12345-192.168.204.33@tcp:101:touch.0 00010000:00010000:3.0:1783520933.381636:0:6494:0:(ldlm_lockd.c:1265:ldlm_handle_enqueue()) ### server-side enqueue handler START 00010000:00010000:3.0:1783520933.381648:0:6494:0:(ldlm_lockd.c:1343:ldlm_handle_enqueue()) ### server-side enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024f6b lrc: 2/0,0 mode: --/CR res: [0x200000402:0x728:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT gid 0 flags: 0x40000000000000 nid: local remote: 0xc275876b859f345e expref: -99 pid: 6494 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.381683:0:6494:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024f6b lrc: 3/0,0 mode: CR/CR res: [0x200000402:0x728:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x50000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f345e expref: 64 pid: 6494 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.381705:0:6494:0:(ldlm_lockd.c:1513:ldlm_handle_enqueue()) ### server-side enqueue handler, sending reply (err=0, rc=0) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784de00/0xd5401cbabc024f6b lrc: 3/0,0 mode: CR/CR res: [0x200000402:0x728:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f345e expref: 64 pid: 6494 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.381726:0:6494:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x728:0x0].0x0 (ffff94350a23bf00) 00010000:00010000:3.0:1783520933.381729:0:6494:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 8 00010000:00010000:3.0:1783520933.381732:0:6494:0:(ldlm_lockd.c:1592:ldlm_handle_enqueue()) ### server-side enqueue handler END (lock ffff94350784de00, rc 0) 00000100:00100000:3.0:1783520933.381763:0:6494:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515f0a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+64:251198:x1870153002357888:12345-192.168.204.33@tcp:101:touch.0 Request processed in 152us (275us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520933.381768:0:6494:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98386 00000100:00100000:3.0:1783520933.382933:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002358016 00000100:00100000:3.0:1783520933.382938:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002358016 00000100:00100000:3.0:1783520933.382986:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 44070 00000100:00100000:3.0:1783520933.382996:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515f08000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+6:2387:x1870153002358016:12345-192.168.204.33@tcp:2:touch.0 00000100:00100000:3.0:1783520933.383176:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515f08000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+7:2387:x1870153002358016:12345-192.168.204.33@tcp:2:touch.0 Request processed in 181us (301us total) trans 4294977551 rc 0/0 00000100:00100000:3.0:1783520933.383184:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 44070 00000100:00100000:3.0:1783520933.398768:0:35279:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002358912 00000100:00100000:3.0:1783520933.398777:0:35279:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002358912 00000100:00100000:3.0:1783520933.398842:0:35279:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98390 00000100:00100000:3.0:1783520933.398854:0:35279:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e470e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_010:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+66:251199:x1870153002358912:12345-192.168.204.33@tcp:36:lfs.0 00010000:00010000:3.0:1783520933.398903:0:35279:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520933.398924:0:35279:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbf6a00/0xd5401cbabc024fa3 lrc: 0/0,0 mode: --/PR res: [0x200000402:0x726:0x0].0x0 bits 0x13/0x0 rrc: 6 type: IBT gid 0 flags: 0x44a01000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3473 expref: 66 pid: 17758 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.398940:0:35279:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x726:0x0].0x0 (ffff9435163beb00) 00010000:00010000:3.0:1783520933.398944:0:35279:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520933.398988:0:35279:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024faa lrc: 3/0,1 mode: --/CW res: [0x200000402:0x726:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x40000000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.398999:0:35279:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024faa lrc: 3/0,1 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.399008:0:35279:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024faa lrc: 3/0,1 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.399021:0:35279:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ce00/0xd5401cbabc024fb1 lrc: 3/0,1 mode: --/PW res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x0/0x0 rrc: 2 type: IBT flags: 0x40000000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.399026:0:35279:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ce00/0xd5401cbabc024fb1 lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.399032:0:35279:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ce00/0xd5401cbabc024fb1 lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.399481:0:35279:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784d000/0xd5401cbabc024fb8 lrc: 3/0,1 mode: --/PW res: [0x200000402:0x729:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x40000000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.399490:0:35279:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94350784d000/0xd5401cbabc024fb8 lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x729:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.399497:0:35279:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94350784d000/0xd5401cbabc024fb8 lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x729:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00000004:00010000:3.0:1783520933.399510:0:35279:0:(mdt_handler.c:4478:mdt_save_lock()) ### save lock request 0000000002f84de6 reply state 00000000b0f5709a transno 8589957884 ns: mdt-lustre-MDT0000_UUID lock: ffff94350784d000/0xd5401cbabc024fb8 lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x729:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.399520:0:35279:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94350784d000/0xd5401cbabc024fb8 lrc: 3/0,1 mode: TXN/TXN res: [0x200000402:0x729:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.399525:0:35279:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x729:0x0].0x0 (ffff943508e9ee00) 00010000:00010000:3.0:1783520933.399528:0:35279:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00000100:00100000:3.0:1783520933.399530:0:35279:0:(service.c:179:ptlrpc_save_lock()) nlocks 0 00000004:00200000:3.0:1783520933.399534:0:35279:0:(mdt_handler.c:909:mdt_pack_attr2body()) [0x200000402:0x729:0x0]: returning size 4096 00010000:00010000:3.0:1783520933.399555:0:35279:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024faa lrc: 4/0,1 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.399562:0:35279:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024faa lrc: 4/0,1 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.399568:0:35279:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024faa lrc: 3/0,0 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.399573:0:35279:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024faa lrc: 4/0,0 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.399578:0:35279:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784c200 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520933.399582:0:35279:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024faa lrc: 4/0,0 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.399589:0:35279:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024faa lrc: 5/0,0 mode: CW/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40218400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.399597:0:35279:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x726:0x0].0x0 (ffff9435163beb00) 00010000:00010000:3.0:1783520933.399599:0:35279:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520933.399602:0:35279:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024faa lrc: 3/0,0 mode: --/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x44a19400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.399607:0:35279:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024faa lrc: 0/0,0 mode: --/CW res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x44a19400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00000004:00010000:3.0:1783520933.399618:0:35279:0:(mdt_handler.c:4478:mdt_save_lock()) ### save lock request 0000000002f84de6 reply state 00000000b0f5709a transno 8589957884 ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ce00/0xd5401cbabc024fb1 lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.399625:0:35279:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ce00/0xd5401cbabc024fb1 lrc: 3/0,1 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.399630:0:35279:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x726:0x0].0xfb53d8db (ffff943508e9e100) 00010000:00010000:3.0:1783520933.399632:0:35279:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00000100:00100000:3.0:1783520933.399634:0:35279:0:(service.c:179:ptlrpc_save_lock()) nlocks 1 00000100:00100000:3.0:1783520933.399698:0:35279:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e470e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_010:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+67:251199:x1870153002358912:12345-192.168.204.33@tcp:36:lfs.0 Request processed in 848us (989us total) trans 8589957884 rc 0/0 00000100:00100000:3.0:1783520933.399706:0:35279:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98390 00000100:00100000:3.0:1783520933.415428:0:6494:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002359424 00000100:00100000:3.0:1783520933.415434:0:6494:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002359424 00000100:00100000:3.0:1783520933.415472:0:6494:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98393 00000100:00100000:3.0:1783520933.415482:0:6494:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+69:251201:x1870153002359424:12345-192.168.204.33@tcp:101:rm.0 00010000:00010000:3.0:1783520933.415494:0:6494:0:(ldlm_lockd.c:1265:ldlm_handle_enqueue()) ### server-side enqueue handler START 00010000:00010000:3.0:1783520933.415508:0:6494:0:(ldlm_lockd.c:1343:ldlm_handle_enqueue()) ### server-side enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024fe2 lrc: 2/0,0 mode: --/CR res: [0x200000402:0x726:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT gid 0 flags: 0x40000000000000 nid: local remote: 0xc275876b859f3481 expref: -99 pid: 6494 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.415599:0:6494:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784da00/0xd5401cbabc024fe9 lrc: 3/1,0 mode: --/CR res: [0x200000402:0x726:0x0].0x0 bits 0x0/0x0 rrc: 6 type: IBT flags: 0x40000000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.415611:0:6494:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94350784da00/0xd5401cbabc024fe9 lrc: 3/1,0 mode: CR/CR res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x50210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.415620:0:6494:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94350784da00/0xd5401cbabc024fe9 lrc: 3/1,0 mode: CR/CR res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x40210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.415630:0:6494:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ca00/0xd5401cbabc024ff0 lrc: 3/1,0 mode: --/PR res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x0/0x0 rrc: 4 type: IBT flags: 0x40000000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.415637:0:6494:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ca00/0xd5401cbabc024ff0 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 4 type: IBT flags: 0x50210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.415643:0:6494:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ca00/0xd5401cbabc024ff0 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.415670:0:6494:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784da00/0xd5401cbabc024fe9 lrc: 4/1,0 mode: CR/CR res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x40210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.415677:0:6494:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784da00/0xd5401cbabc024fe9 lrc: 4/1,0 mode: CR/CR res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x50210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.415683:0:6494:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff94350784da00/0xd5401cbabc024fe9 lrc: 3/0,0 mode: CR/CR res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x50210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.415689:0:6494:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff94350784da00/0xd5401cbabc024fe9 lrc: 4/0,0 mode: CR/CR res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x40210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.415698:0:6494:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784da00 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520933.415702:0:6494:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784da00/0xd5401cbabc024fe9 lrc: 4/0,0 mode: CR/CR res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x40210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.415708:0:6494:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784da00/0xd5401cbabc024fe9 lrc: 5/0,0 mode: CR/CR res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x40218400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.415719:0:6494:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x726:0x0].0x0 (ffff9435163beb00) 00010000:00010000:3.0:1783520933.415722:0:6494:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520933.415726:0:6494:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff94350784da00/0xd5401cbabc024fe9 lrc: 3/0,0 mode: --/CR res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x44a19400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.415731:0:6494:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94350784da00/0xd5401cbabc024fe9 lrc: 0/0,0 mode: --/CR res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x44a19400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.415743:0:6494:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ca00/0xd5401cbabc024ff0 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 4 type: IBT flags: 0x50210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.415749:0:6494:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ca00/0xd5401cbabc024ff0 lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 4 type: IBT flags: 0x50210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.415754:0:6494:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ca00/0xd5401cbabc024ff0 lrc: 3/0,0 mode: PR/PR res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.415759:0:6494:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784ca00 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520933.415762:0:6494:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ca00/0xd5401cbabc024ff0 lrc: 3/0,0 mode: PR/PR res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40210400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.415769:0:6494:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ca00/0xd5401cbabc024ff0 lrc: 4/0,0 mode: PR/PR res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40218400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.415780:0:6494:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x726:0x0].0xfb53d8db (ffff943508e9e100) 00010000:00010000:3.0:1783520933.415783:0:6494:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520933.415785:0:6494:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ca00/0xd5401cbabc024ff0 lrc: 2/0,0 mode: --/PR res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 4 type: IBT flags: 0x44a19400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.415792:0:6494:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ca00/0xd5401cbabc024ff0 lrc: 0/0,0 mode: --/PR res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 4 type: IBT flags: 0x44a19400000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.415826:0:6494:0:(ldlm_lockd.c:1513:ldlm_handle_enqueue()) ### server-side enqueue handler, sending reply (err=301, rc=0) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024fe2 lrc: 1/0,0 mode: --/CR res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x44000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3481 expref: 70 pid: 6494 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.415835:0:6494:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x726:0x0].0x0 (ffff9435163beb00) 00010000:00010000:3.0:1783520933.415837:0:6494:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520933.415841:0:6494:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94350784c200/0xd5401cbabc024fe2 lrc: 0/0,0 mode: --/CR res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x44801000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3481 expref: 70 pid: 6494 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.415850:0:6494:0:(ldlm_lockd.c:1592:ldlm_handle_enqueue()) ### server-side enqueue handler END (lock ffff94350784c200, rc 0) 00000100:00100000:3.0:1783520933.415894:0:6494:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+69:251201:x1870153002359424:12345-192.168.204.33@tcp:101:rm.0 Request processed in 414us (498us total) trans 0 rc 301/301 00000100:00100000:3.0:1783520933.415901:0:6494:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98393 00000100:00100000:3.0:1783520933.423872:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002359680 00000100:00100000:3.0:1783520933.423877:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002359680 00000100:00100000:3.0:1783520933.423909:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 44071 00000100:00100000:3.0:1783520933.423917:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+6:2388:x1870153002359680:12345-192.168.204.33@tcp:101:rm.0 00010000:00010000:3.0:1783520933.423927:0:178496:0:(ldlm_lockd.c:1265:ldlm_handle_enqueue()) ### server-side enqueue handler START 00010000:00010000:3.0:1783520933.423943:0:178496:0:(ldlm_lockd.c:1343:ldlm_handle_enqueue()) ### server-side enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000000fa48b86/0xd5401cbabc025013 lrc: 2/0,0 mode: --/PR res: [0x280000401:0x917:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0xc275876b859f348f expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00010000:3.0:1783520933.423969:0:178496:0:(ofd_lvb.c:151:ofd_lvbo_init()) res: [0x280000401:0x917:0x0] initial LVB size: 0, mtime: 0x6a4e5ea5, atime: 0x6a4e5ea5, ctime: 0x6a4e5ea5, blocks: 0x0 00010000:00010000:3.0:1783520933.423984:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x917:0x0].0x0 since=4128s v=(0 0) 00010000:00010000:3.0:1783520933.423989:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x917:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520933.423997:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000000fa48b86/0xd5401cbabc025013 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x917:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f348f expref: 7 pid: 178496 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.424013:0:178496:0:(ldlm_lockd.c:1513:ldlm_handle_enqueue()) ### server-side enqueue handler, sending reply (err=0, rc=0) ns: filter-lustre-OST0000_UUID lock: 000000000fa48b86/0xd5401cbabc025013 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x917:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f348f expref: 7 pid: 178496 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520933.424024:0:178496:0:(ldlm_lockd.c:1592:ldlm_handle_enqueue()) ### server-side enqueue handler END (lock ffff94351cbf5c00, rc 0) 00000100:00100000:3.0:1783520933.424061:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+7:2388:x1870153002359680:12345-192.168.204.33@tcp:101:rm.0 Request processed in 145us (334us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520933.424065:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 44071 00000100:00100000:3.0:1783520933.813641:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e004a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3627:1870152993050880:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1783520933.813700:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520933.813935:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e004a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3627:1870152993050880:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1783520935.349623:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e004a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3628:1870152993051008:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1783520935.349693:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.349948:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e004a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3628:1870152993051008:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1783520935.414399:0:17758:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002360320 00000100:00100000:3.0:1783520935.414408:0:17758:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002360320 00000100:00100000:3.0:1783520935.414474:0:17758:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 7076 00000100:00100000:3.0:1783520935.414491:0:17758:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+17:2387:x1870153002360320:12345-192.168.204.33@tcp:400:kworker.0 00000100:00100000:3.0:1783520935.414553:0:17758:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+17:2387:x1870153002360320:12345-192.168.204.33@tcp:400:kworker.0 Request processed in 65us (214us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.414566:0:17758:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 7076 00000100:00100000:3.0:1783520935.414602:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002360448 00000100:00100000:3.0:1783520935.414618:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002360448 00000100:00100000:3.0:1783520935.414660:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002360576 00000100:00100000:3.0:1783520935.414665:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002360576 00000100:00100000:3.0:1783520935.414696:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002360960 00000100:00100000:3.0:1783520935.414701:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002360960 00000100:00100000:3.0:1783520935.414737:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 4756 00000100:00100000:3.0:1783520935.414833:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+7:2388:x1870153002360448:12345-192.168.204.33@tcp:400:kworker.0 00000100:00100000:3.0:1783520935.415319:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+7:2388:x1870153002360448:12345-192.168.204.33@tcp:400:kworker.0 Request processed in 263us (607us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.415328:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 4756 00000001:02000400:3.0:1783520935.622791:0:179290:0:(debug.c:654:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanityn test 71a: correct file map just after write operation is finished ========================================================== 10:28:55 (1783520935) 00010000:00010000:3.0:1783520935.702731:0:3611:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(TXN) ns: mdt-lustre-MDT0000_UUID lock: ffff943512184a00/0xd5401cbabc024cb6 lrc: 3/0,1 mode: TXN/TXN res: [0x200000007:0x1:0x0].0x121d381d bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.702744:0:3611:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff943512184a00/0xd5401cbabc024cb6 lrc: 2/0,0 mode: TXN/TXN res: [0x200000007:0x1:0x0].0x121d381d bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.702752:0:3611:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff943512184a00/0xd5401cbabc024cb6 lrc: 3/0,0 mode: TXN/TXN res: [0x200000007:0x1:0x0].0x121d381d bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.702761:0:3611:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184a00 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520935.702764:0:3611:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943512184a00/0xd5401cbabc024cb6 lrc: 3/0,0 mode: TXN/TXN res: [0x200000007:0x1:0x0].0x121d381d bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.702771:0:3611:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943512184a00/0xd5401cbabc024cb6 lrc: 4/0,0 mode: TXN/TXN res: [0x200000007:0x1:0x0].0x121d381d bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40218400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.702782:0:3611:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x121d381d (ffff9435163be500) 00010000:00010000:3.0:1783520935.702784:0:3611:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520935.702786:0:3611:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff943512184a00/0xd5401cbabc024cb6 lrc: 2/0,0 mode: --/TXN res: [0x200000007:0x1:0x0].0x121d381d bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.702791:0:3611:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943512184a00/0xd5401cbabc024cb6 lrc: 0/0,0 mode: --/TXN res: [0x200000007:0x1:0x0].0x121d381d bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.702810:0:3611:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(TXN) ns: mdt-lustre-MDT0000_UUID lock: ffff943512184200/0xd5401cbabc024cbd lrc: 3/0,1 mode: TXN/TXN res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x50210000000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.702815:0:3611:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff943512184200/0xd5401cbabc024cbd lrc: 2/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x50210400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.702819:0:3611:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff943512184200/0xd5401cbabc024cbd lrc: 3/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40210400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.702822:0:3611:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520935.702825:0:3611:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943512184200/0xd5401cbabc024cbd lrc: 3/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40210400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.702830:0:3611:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943512184200/0xd5401cbabc024cbd lrc: 4/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40218400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.702839:0:3611:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x726:0x0].0x0 (ffff9435163beb00) 00010000:00010000:3.0:1783520935.702843:0:3611:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520935.702845:0:3611:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff943512184200/0xd5401cbabc024cbd lrc: 2/0,0 mode: --/TXN res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x44a19400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.702850:0:3611:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943512184200/0xd5401cbabc024cbd lrc: 0/0,0 mode: --/TXN res: [0x200000402:0x726:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x44a19400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.702866:0:3611:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(TXN) ns: mdt-lustre-MDT0000_UUID lock: ffff943512184800/0xd5401cbabc024cfc lrc: 3/0,1 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210000000000 pid: 35281 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.702873:0:3611:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff943512184800/0xd5401cbabc024cfc lrc: 2/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210400000000 pid: 35281 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.702884:0:3611:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff943512184800/0xd5401cbabc024cfc lrc: 3/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210400000000 pid: 35281 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.702897:0:3611:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184800 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520935.702903:0:3611:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943512184800/0xd5401cbabc024cfc lrc: 3/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210400000000 pid: 35281 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.702912:0:3611:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943512184800/0xd5401cbabc024cfc lrc: 4/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40218400000000 pid: 35281 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.702923:0:3611:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x726:0x0].0xb1bed465 (ffff9435163be900) 00010000:00010000:3.0:1783520935.702930:0:3611:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520935.702937:0:3611:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff943512184800/0xd5401cbabc024cfc lrc: 2/0,0 mode: --/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x44a19400000000 pid: 35281 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.702948:0:3611:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943512184800/0xd5401cbabc024cfc lrc: 0/0,0 mode: --/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x44a19400000000 pid: 35281 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.702963:0:3611:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(TXN) ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfac00/0xd5401cbabc024d11 lrc: 3/0,1 mode: TXN/TXN res: [0x200000402:0x727:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.702970:0:3611:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfac00/0xd5401cbabc024d11 lrc: 2/0,0 mode: TXN/TXN res: [0x200000402:0x727:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210400000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.702983:0:3611:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfac00/0xd5401cbabc024d11 lrc: 3/0,0 mode: TXN/TXN res: [0x200000402:0x727:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.702990:0:3611:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfac00 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520935.702994:0:3611:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfac00/0xd5401cbabc024d11 lrc: 3/0,0 mode: TXN/TXN res: [0x200000402:0x727:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703004:0:3611:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfac00/0xd5401cbabc024d11 lrc: 4/0,0 mode: TXN/TXN res: [0x200000402:0x727:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40218400000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703016:0:3611:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x727:0x0].0x0 (ffff94350a23b500) 00010000:00010000:3.0:1783520935.703022:0:3611:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520935.703027:0:3611:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfac00/0xd5401cbabc024d11 lrc: 2/0,0 mode: --/TXN res: [0x200000402:0x727:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703034:0:3611:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfac00/0xd5401cbabc024d11 lrc: 0/0,0 mode: --/TXN res: [0x200000402:0x727:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703060:0:3611:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(TXN) ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfb200/0xd5401cbabc024f2c lrc: 3/0,1 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x50210000000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703071:0:3611:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfb200/0xd5401cbabc024f2c lrc: 2/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x50210400000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703080:0:3611:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfb200/0xd5401cbabc024f2c lrc: 3/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40210400000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703087:0:3611:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb200 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520935.703091:0:3611:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfb200/0xd5401cbabc024f2c lrc: 3/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40210400000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703112:0:3611:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfb200/0xd5401cbabc024f2c lrc: 4/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40218400000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703124:0:3611:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x726:0x0].0xb1bed465 (ffff9435163be900) 00010000:00010000:3.0:1783520935.703128:0:3611:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520935.703132:0:3611:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfb200/0xd5401cbabc024f2c lrc: 2/0,0 mode: --/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x44a19400000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703138:0:3611:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfb200/0xd5401cbabc024f2c lrc: 0/0,0 mode: --/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x44a19400000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703153:0:3611:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(TXN) ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbc00/0xd5401cbabc024f4f lrc: 3/0,1 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x50210000000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703159:0:3611:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbc00/0xd5401cbabc024f4f lrc: 2/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x50210400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703174:0:3611:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbc00/0xd5401cbabc024f4f lrc: 3/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x40210400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703181:0:3611:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfbc00 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520935.703184:0:3611:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbc00/0xd5401cbabc024f4f lrc: 3/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x40210400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703190:0:3611:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbc00/0xd5401cbabc024f4f lrc: 4/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x40218400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703206:0:3611:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x726:0x0].0xb1bed465 (ffff9435163be900) 00010000:00010000:3.0:1783520935.703213:0:3611:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520935.703217:0:3611:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbc00/0xd5401cbabc024f4f lrc: 2/0,0 mode: --/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x44a19400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703227:0:3611:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbc00/0xd5401cbabc024f4f lrc: 0/0,0 mode: --/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x44a19400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703330:0:3611:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(TXN) ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfb800/0xd5401cbabc024f64 lrc: 3/0,1 mode: TXN/TXN res: [0x200000402:0x728:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 7857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703347:0:3611:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfb800/0xd5401cbabc024f64 lrc: 2/0,0 mode: TXN/TXN res: [0x200000402:0x728:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210400000000 pid: 7857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703385:0:3611:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfb800/0xd5401cbabc024f64 lrc: 3/0,0 mode: TXN/TXN res: [0x200000402:0x728:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 7857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703392:0:3611:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb800 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520935.703395:0:3611:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfb800/0xd5401cbabc024f64 lrc: 3/0,0 mode: TXN/TXN res: [0x200000402:0x728:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 7857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703403:0:3611:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfb800/0xd5401cbabc024f64 lrc: 4/0,0 mode: TXN/TXN res: [0x200000402:0x728:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40218400000000 pid: 7857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703422:0:3611:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x728:0x0].0x0 (ffff94350a23bf00) 00010000:00010000:3.0:1783520935.703428:0:3611:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520935.703434:0:3611:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfb800/0xd5401cbabc024f64 lrc: 2/0,0 mode: --/TXN res: [0x200000402:0x728:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 7857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703457:0:3611:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfb800/0xd5401cbabc024f64 lrc: 0/0,0 mode: --/TXN res: [0x200000402:0x728:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 7857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703484:0:3611:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(TXN) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ce00/0xd5401cbabc024fb1 lrc: 3/0,1 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 3 type: IBT flags: 0x50210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703521:0:3611:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ce00/0xd5401cbabc024fb1 lrc: 2/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 3 type: IBT flags: 0x50210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703535:0:3611:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ce00/0xd5401cbabc024fb1 lrc: 3/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 3 type: IBT flags: 0x40210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703554:0:3611:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784ce00 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520935.703563:0:3611:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ce00/0xd5401cbabc024fb1 lrc: 3/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 3 type: IBT flags: 0x40210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703588:0:3611:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ce00/0xd5401cbabc024fb1 lrc: 4/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 3 type: IBT flags: 0x40218400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703614:0:3611:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x726:0x0].0xfb53d8db (ffff943508e9e100) 00010000:00010000:3.0:1783520935.703620:0:3611:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520935.703627:0:3611:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ce00/0xd5401cbabc024fb1 lrc: 2/0,0 mode: --/TXN res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 3 type: IBT flags: 0x44a19400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703645:0:3611:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94350784ce00/0xd5401cbabc024fb1 lrc: 0/0,0 mode: --/TXN res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 3 type: IBT flags: 0x44a19400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703664:0:3611:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(TXN) ns: mdt-lustre-MDT0000_UUID lock: ffff94350784d000/0xd5401cbabc024fb8 lrc: 3/0,1 mode: TXN/TXN res: [0x200000402:0x729:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703675:0:3611:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff94350784d000/0xd5401cbabc024fb8 lrc: 2/0,0 mode: TXN/TXN res: [0x200000402:0x729:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703681:0:3611:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff94350784d000/0xd5401cbabc024fb8 lrc: 3/0,0 mode: TXN/TXN res: [0x200000402:0x729:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703689:0:3611:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94350784d000 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520935.703692:0:3611:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784d000/0xd5401cbabc024fb8 lrc: 3/0,0 mode: TXN/TXN res: [0x200000402:0x729:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703710:0:3611:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94350784d000/0xd5401cbabc024fb8 lrc: 4/0,0 mode: TXN/TXN res: [0x200000402:0x729:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40218400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703722:0:3611:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x729:0x0].0x0 (ffff943508e9ee00) 00010000:00010000:3.0:1783520935.703728:0:3611:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520935.703733:0:3611:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff94350784d000/0xd5401cbabc024fb8 lrc: 2/0,0 mode: --/TXN res: [0x200000402:0x729:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703741:0:3611:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94350784d000/0xd5401cbabc024fb8 lrc: 0/0,0 mode: --/TXN res: [0x200000402:0x729:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703766:0:3611:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(TXN) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7e00/0xd5401cbabc024fc6 lrc: 3/0,1 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 10322 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703780:0:3611:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7e00/0xd5401cbabc024fc6 lrc: 2/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210400000000 pid: 10322 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703793:0:3611:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7e00/0xd5401cbabc024fc6 lrc: 3/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 10322 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703799:0:3611:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7e00 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520935.703803:0:3611:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7e00/0xd5401cbabc024fc6 lrc: 3/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 10322 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703809:0:3611:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7e00/0xd5401cbabc024fc6 lrc: 4/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40218400000000 pid: 10322 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703819:0:3611:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x726:0x0].0xfb53d8db (ffff943508e9e100) 00010000:00010000:3.0:1783520935.703824:0:3611:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520935.703830:0:3611:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7e00/0xd5401cbabc024fc6 lrc: 2/0,0 mode: --/TXN res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 10322 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703841:0:3611:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7e00/0xd5401cbabc024fc6 lrc: 0/0,0 mode: --/TXN res: [0x200000402:0x726:0x0].0xfb53d8db bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 10322 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703861:0:3611:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(TXN) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc025021 lrc: 3/0,1 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 35273 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703883:0:3611:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc025021 lrc: 2/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210400000000 pid: 35273 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703906:0:3611:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc025021 lrc: 3/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 35273 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703924:0:3611:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6800 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520935.703932:0:3611:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc025021 lrc: 3/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 35273 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703952:0:3611:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc025021 lrc: 4/0,0 mode: TXN/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40218400000000 pid: 35273 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.703979:0:3611:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x726:0x0].0xb1bed465 (ffff9435163be900) 00010000:00010000:3.0:1783520935.703987:0:3611:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520935.703994:0:3611:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc025021 lrc: 2/0,0 mode: --/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 35273 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.704018:0:3611:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc025021 lrc: 0/0,0 mode: --/TXN res: [0x200000402:0x726:0x0].0xb1bed465 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 35273 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.704041:0:3611:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(TXN) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6c00/0xd5401cbabc025044 lrc: 3/0,1 mode: TXN/TXN res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 6493 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.704060:0:3611:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6c00/0xd5401cbabc025044 lrc: 2/0,0 mode: TXN/TXN res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210400000000 pid: 6493 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.704082:0:3611:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6c00/0xd5401cbabc025044 lrc: 3/0,0 mode: TXN/TXN res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 6493 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.704099:0:3611:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6c00 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520935.704106:0:3611:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6c00/0xd5401cbabc025044 lrc: 3/0,0 mode: TXN/TXN res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 6493 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.704119:0:3611:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6c00/0xd5401cbabc025044 lrc: 4/0,0 mode: TXN/TXN res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40218400000000 pid: 6493 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.704135:0:3611:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x5ba6781 (ffff943507a8dc00) 00010000:00010000:3.0:1783520935.704140:0:3611:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520935.704142:0:3611:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6c00/0xd5401cbabc025044 lrc: 2/0,0 mode: --/TXN res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 6493 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.704147:0:3611:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6c00/0xd5401cbabc025044 lrc: 0/0,0 mode: --/TXN res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 6493 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.704268:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.704329:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1783520935.704452:0:6489:0:(pinger.c:500:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1783520935.704527:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc025091 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x917:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.704541:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0000_UUID lock: 000000000fa48b86/0xd5401cbabc025013 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x917:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f348f expref: 6 pid: 178496 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.704548:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x917:0x0].0x0 since=2s v=(0 0) 00010000:00010000:1.0:1783520935.704551:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x917:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.704555:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc025091 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x917:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.704577:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0000_UUID lock: 000000000fa48b86/0xd5401cbabc025013 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x917:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f348f expref: 6 pid: 178496 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.704588:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0000_UUID lock: 000000000fa48b86/0xd5401cbabc025013 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x917:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f348f expref: 6 pid: 178496 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.704606:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c5f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0000_UUID:178496:1870152993053568:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.704659:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6540 going to sleep for 11 seconds 00000001:00080000:3.0:1783520935.705302:0:7422:0:(tgt_lastrcvd.c:983:tgt_cb_last_committed()) lustre-MDT0001: transno 4294985692 is committed 00000001:00080000:3.0:1783520935.705317:0:7422:0:(tgt_lastrcvd.c:983:tgt_cb_last_committed()) lustre-MDT0001: transno 4294985691 is committed 00000020:00080000:3.0:1783520935.705353:0:7422:0:(update_trans.c:60:top_multiple_thandle_dump()) lustre-MDT0001-osd tmt ffff943512af1b40 refcount 2 committed 0 result 0 batchid 3480 00000020:00080000:3.0:1783520935.705379:0:7422:0:(update_trans.c:68:top_multiple_thandle_dump()) st ffff943618fde400 obd lustre-MDT0000-osp-MDT0001 committed 1 started 1 stopped 1 result 0 sub_th ffff943512af1840 00000020:00080000:3.0:1783520935.705383:0:7422:0:(update_trans.c:73:top_multiple_thandle_dump()) cookie [0x200000401:0x3:0x0].1507 00000020:00080000:3.0:1783520935.705385:0:7422:0:(update_trans.c:68:top_multiple_thandle_dump()) st ffff943502ee2300 obd lustre-MDT0001-osd committed 1 started 1 stopped 1 result 0 sub_th ffff943514ebbe00 00000020:00080000:3.0:1783520935.705389:0:7422:0:(update_trans.c:73:top_multiple_thandle_dump()) cookie [0x240000400:0x3:0x0].1166 00000100:00100000:3.0:1783520935.706419:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993052672:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.706432:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.706441:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025060 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x520:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.706473:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002361728 00000100:00100000:3.0:1783520935.706479:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002361728 00010000:00010000:3.0:1783520935.706520:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000075f15121/0xd5401cbabc01f1af lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x521:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef8b4 expref: 990 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.706522:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c550700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993052928:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.706543:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc02506e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x523:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.706561:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35666 00000100:00100000:3.0:1783520935.706571:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad0000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+990:2388:x1870153002361728:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.706582:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.706587:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000075f15121/0xd5401cbabc01f1af lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x521:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef8b4 expref: 990 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.706598:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000075f15121/0xd5401cbabc01f1af lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x521:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef8b4 expref: 990 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.706619:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000075f15121/0xd5401cbabc01f1af lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x521:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef8b4 expref: 990 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.706632:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x521:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.706636:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x521:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.706640:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc025075 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x521:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.706654:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc025075 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x521:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.706677:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.706728:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad0000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+989:2388:x1870153002361728:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 152us (295us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.706743:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35666 00010000:00010000:1.0:1783520935.706788:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.706795:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc025075 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x521:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.706809:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf6e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.706814:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc025075 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x521:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.706824:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc025075 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x521:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.706837:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc025075 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x521:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.706843:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc025075 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x521:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.706861:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.706864:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.706866:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520935.706947:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ae4380/2, rc 0, transno 4294975574 00000100:00100000:1.0:1783520935.706964:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ae4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993051520:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.707044:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ace680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993053696:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.707078:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.707197:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002361856 00000100:00100000:1.0:1783520935.707204:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002361856 00010000:00010000:1.0:1783520935.707229:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000094eabe83/0xd5401cbabc01f193 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x51f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef898 expref: 989 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.707238:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002362112 00000100:00100000:3.0:1783520935.707245:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002362112 00000100:00100000:1.0:1783520935.707284:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002361984 00000100:00100000:1.0:1783520935.707289:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002361984 00010000:00010000:1.0:1783520935.707306:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f9e5bb00/0xd5401cbabc01f1a1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x520:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef8a6 expref: 990 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.707335:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002362240 00000100:00100000:1.0:1783520935.707340:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002362240 00010000:00010000:3.0:1783520935.707378:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000003242edc/0xd5401cbabc01f1d9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x524:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef8de expref: 992 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.707390:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000de420386/0xd5401cbabc01f1cb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x523:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef8d0 expref: 992 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.707410:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35667 00000100:00100000:3.0:1783520935.707420:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517244a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+992:2387:x1870153002361856:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.707432:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.707433:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35669 00010000:00010000:3.0:1783520935.707443:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000094eabe83/0xd5401cbabc01f193 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x51f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef898 expref: 992 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.707449:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517246a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+992:2387:x1870153002362112:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.707465:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.707475:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000003242edc/0xd5401cbabc01f1d9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x524:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef8de expref: 992 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.707510:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000003242edc/0xd5401cbabc01f1d9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x524:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef8de expref: 992 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.707533:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000003242edc/0xd5401cbabc01f1d9 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x524:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef8de expref: 991 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.707557:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x524:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.707566:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x524:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.707577:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000059c8dff3/0xd5401cbabc025067 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x524:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.707582:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000094eabe83/0xd5401cbabc01f193 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x51f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef898 expref: 990 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.707597:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000059c8dff3/0xd5401cbabc025067 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x524:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.707610:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000094eabe83/0xd5401cbabc01f193 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x51f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef898 expref: 990 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.707627:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.707643:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x51f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.707648:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x51f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.707654:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc025059 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x51f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.707656:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517246a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+988:2387:x1870153002362112:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 210us (438us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.707669:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc025059 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x51f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.707676:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35669 00000100:00100000:1.0:1783520935.707699:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002362624 00010000:00010000:3.0:1783520935.707702:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.707708:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002362624 00010000:00010000:1.0:1783520935.707731:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000076cc9d5a/0xd5401cbabc01f1bd lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x522:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef8c2 expref: 990 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.707765:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002362880 00000100:00100000:1.0:1783520935.707770:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002362880 00010000:00010000:1.0:1783520935.707791:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0000_UUID lock: 000000000fa48b86/0xd5401cbabc025013 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x917:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f348f expref: 7 pid: 178496 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.707824:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35671 00000100:00100000:3.0:1783520935.707827:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517244a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+990:2387:x1870153002361856:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 405us (645us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.707837:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517245c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+990:2387:x1870153002362496:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.707845:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.707852:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f66b051f/0xd5401cbabc01f1e7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x525:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef8ec expref: 990 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.707865:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f66b051f/0xd5401cbabc01f1e7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x525:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef8ec expref: 989 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.707883:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f66b051f/0xd5401cbabc01f1e7 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x525:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef8ec expref: 989 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.707895:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x525:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.707901:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x525:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.707906:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02507c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x525:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.707911:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02507c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x525:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.707929:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.707931:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35667 00000100:00100000:1.0:1783520935.707959:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517245c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+987:2387:x1870153002362496:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 122us (473us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.707983:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35671 00000100:00100000:1.0:1783520935.708009:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35673 00000100:00100000:1.0:1783520935.708022:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517245f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+985:2389:x1870153002362752:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.708034:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.708044:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000051bdbdba/0xd5401cbabc022ab2 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x526:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31a2 expref: 985 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.708065:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35674 00010000:00010000:1.0:1783520935.708075:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000051bdbdba/0xd5401cbabc022ab2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x526:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31a2 expref: 985 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.708082:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517245180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+7:2387:x1870153002362880:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.708101:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.708109:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0000_UUID lock: 000000000fa48b86/0xd5401cbabc025013 lrc: 5/0,0 mode: PR/PR res: [0x280000401:0x917:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f348f expref: 7 pid: 178496 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.708111:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000051bdbdba/0xd5401cbabc022ab2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x526:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31a2 expref: 985 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.708137:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x526:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.708143:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x526:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.708148:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02508a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x526:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.708154:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02508a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x526:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.708170:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.708194:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517245f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+983:2389:x1870153002362752:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 174us (647us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.708200:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0000_UUID lock: 000000000fa48b86/0xd5401cbabc025013 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x917:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f348f expref: 7 pid: 178496 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.708207:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35673 00010000:00010000:3.0:1783520935.708229:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x917:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.708234:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x280000401:0x917:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.708238:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc025091 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x917:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.708249:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc025091 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x917:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.708249:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02508a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x526:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.708277:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02508a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x526:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.708285:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.708296:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02508a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x526:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.708316:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517245180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+7:2387:x1870153002362880:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 236us (745us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.708318:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02508a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x526:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.708335:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35674 00010000:00010000:1.0:1783520935.708345:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.708461:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517245500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+15:3627:x1870152993052160:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4288us (4690us total) trans 4294975576 rc 0/0 00000100:00100000:1.0:1783520935.708477:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44079 00010000:00010000:1.0:1783520935.708532:0:178496:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0000_UUID lock: 000000000fa48b86/0xd5401cbabc025013 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x917:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f348f expref: 6 pid: 178496 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.708536:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc025059 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x51f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.708555:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc025059 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x51f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.708556:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c5f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0000_UUID:178496:1870152993053568:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.708572:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc025059 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x51f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.708578:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc025059 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x51f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.708582:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc025091 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x917:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.708594:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc025091 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x917:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.708602:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc025091 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x917:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.708611:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc025091 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x917:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.708616:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.708621:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0000_UUID lock: 000000006847d995/0xd5401cbabc025091 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x917:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.708642:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.708773:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+6:3625:x1870152993051136:12345-0@lo:6:osp-syn-0-0.0 Request processed in 4295us (4446us total) trans 4294977552 rc 0/0 00000100:00100000:1.0:1783520935.708785:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44080 00010000:00010000:1.0:1783520935.708808:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520935.708832:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ae6a00/2, rc 0, transno 4294975577 00000100:00100000:1.0:1783520935.708850:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ae6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993051264:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.708869:0:3625:0:(client.c:3112:ptlrpc_free_committed()) lustre-OST0000-osc-MDT0000: committing for last_committed 4294977550 gen 2 00000004:00080000:1.0:1783520935.708874:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514fdc000, transno 4294976534 00000100:00100000:3.0:1783520935.708876:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+15:3625:x1870152993051264:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5597us (5850us total) trans 4294975577 rc 0/0 00000100:00100000:3.0:1783520935.708888:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44072 00000004:00080000:1.0:1783520935.708889:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514fdce00, transno 4294976535 00000004:00080000:1.0:1783520935.708895:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514fded80, transno 4294976536 00000004:00080000:1.0:1783520935.708900:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514fdf480, transno 4294976537 00000004:00080000:1.0:1783520935.708904:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514fdfb80, transno 4294976538 00000004:00080000:1.0:1783520935.708908:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514fdf100, transno 4294976539 00000100:00100000:3.0:1783520935.708911:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000004:00080000:1.0:1783520935.708912:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514fde300, transno 4294976540 00000004:00080000:1.0:1783520935.708918:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514fdd180, transno 4294976541 00000004:00080000:1.0:1783520935.708923:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e005500, transno 4294976542 00000100:00100000:3.0:1783520935.708925:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517246300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993054080:0@lo:6:osp-syn-1-0.0 00000004:00080000:1.0:1783520935.708929:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363fe55180, transno 4294976543 00000004:00080000:1.0:1783520935.708935:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e007800, transno 4294976544 00000004:00080000:1.0:1783520935.708939:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94361103ad80, transno 4294976545 00000004:00080000:1.0:1783520935.708944:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435032f1180, transno 4294976546 00000004:00080000:1.0:1783520935.708950:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94361103aa00, transno 4294976547 00000004:00080000:1.0:1783520935.708957:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943611038e00, transno 4294976548 00000100:00100000:3.0:1783520935.708958:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:1.0:1783520935.708966:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350544ca80, transno 4294976549 00000004:00080000:1.0:1783520935.708970:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435032f0e00, transno 4294976550 00000004:00080000:1.0:1783520935.708973:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435032f2d80, transno 4294976551 00000004:00080000:1.0:1783520935.708978:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435032f3480, transno 4294976552 00000004:00080000:1.0:1783520935.708986:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350544e300, transno 4294976553 00000004:00080000:1.0:1783520935.708991:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350544f800, transno 4294976554 00000004:00080000:1.0:1783520935.708994:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350544d180, transno 4294976555 00000004:00080000:1.0:1783520935.708999:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363d322300, transno 4294976556 00000004:00080000:1.0:1783520935.709002:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350544fb80, transno 4294976557 00000004:00080000:1.0:1783520935.709005:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363d321c00, transno 4294976558 00000004:00080000:1.0:1783520935.709007:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350544df80, transno 4294976559 00000004:00080000:1.0:1783520935.709011:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435032f2a00, transno 4294976560 00000004:00080000:1.0:1783520935.709015:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363d323800, transno 4294976561 00000004:00080000:1.0:1783520935.709021:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350794dc00, transno 4294976562 00000004:00080000:1.0:1783520935.709027:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350794d500, transno 4294976563 00000004:00080000:1.0:1783520935.709033:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350794d180, transno 4294976564 00000004:00080000:1.0:1783520935.709039:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350794e300, transno 4294976565 00000004:00080000:1.0:1783520935.709044:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350794ed80, transno 4294976566 00000004:00080000:1.0:1783520935.709047:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516683b80, transno 4294976567 00000004:00080000:1.0:1783520935.709050:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350794c000, transno 4294976568 00000004:00080000:1.0:1783520935.709054:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516682a00, transno 4294976569 00000004:00080000:1.0:1783520935.709060:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513139880, transno 4294976570 00000004:00080000:1.0:1783520935.709063:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350544ce00, transno 4294976571 00000004:00080000:1.0:1783520935.709068:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350544dc00, transno 4294976572 00000004:00080000:1.0:1783520935.709073:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351313a680, transno 4294976573 00000004:00080000:1.0:1783520935.709078:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516682d80, transno 4294976574 00000004:00080000:1.0:1783520935.709083:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516680380, transno 4294976575 00000004:00080000:1.0:1783520935.709089:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516683480, transno 4294976576 00000004:00080000:1.0:1783520935.709095:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350544ed80, transno 4294976577 00000004:00080000:1.0:1783520935.709100:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943634570380, transno 4294976578 00000004:00080000:1.0:1783520935.709106:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435032f1c00, transno 4294976579 00000004:00080000:1.0:1783520935.709111:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516680e00, transno 4294976580 00000004:00080000:1.0:1783520935.709115:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943634571f80, transno 4294976581 00000004:00080000:1.0:1783520935.709119:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363d320a80, transno 4294976582 00000004:00080000:1.0:1783520935.709125:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435032f1500, transno 4294976583 00000004:00080000:1.0:1783520935.709131:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363d323480, transno 4294976584 00000004:00080000:1.0:1783520935.709136:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515774a80, transno 4294976585 00000004:00080000:1.0:1783520935.709142:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515774380, transno 4294976586 00000004:00080000:1.0:1783520935.709146:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515774e00, transno 4294976587 00000004:00080000:1.0:1783520935.709151:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b3800, transno 4294976588 00000004:00080000:1.0:1783520935.709155:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b3480, transno 4294976589 00000004:00080000:1.0:1783520935.709158:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b0700, transno 4294976590 00000004:00080000:1.0:1783520935.709161:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435161c0000, transno 4294976591 00000004:00080000:1.0:1783520935.709166:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cc7ce00, transno 4294976592 00000004:00080000:1.0:1783520935.709169:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435161c3800, transno 4294976593 00000004:00080000:1.0:1783520935.709172:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506228a80, transno 4294976594 00000004:00080000:1.0:1783520935.709175:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b2680, transno 4294976595 00000004:00080000:1.0:1783520935.709178:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515777480, transno 4294976596 00000004:00080000:1.0:1783520935.709181:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515476300, transno 4294976597 00000004:00080000:1.0:1783520935.709185:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515476d80, transno 4294976598 00000004:00080000:1.0:1783520935.709190:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943512427b80, transno 4294976599 00000004:00080000:1.0:1783520935.709194:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943512425180, transno 4294976600 00000004:00080000:1.0:1783520935.709200:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943512424700, transno 4294976601 00000004:00080000:1.0:1783520935.709207:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515475500, transno 4294976602 00000004:00080000:1.0:1783520935.709211:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943512426a00, transno 4294976603 00000004:00080000:1.0:1783520935.709215:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943512425c00, transno 4294976604 00000004:00080000:1.0:1783520935.709219:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943512425880, transno 4294976605 00000004:00080000:1.0:1783520935.709225:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515475180, transno 4294976606 00000004:00080000:1.0:1783520935.709231:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515346680, transno 4294976607 00000004:00080000:1.0:1783520935.709239:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515346a00, transno 4294976608 00000004:00080000:1.0:1783520935.709246:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515345c00, transno 4294976609 00000004:00080000:1.0:1783520935.709253:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350d165c00, transno 4294976610 00000004:00080000:1.0:1783520935.709258:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515344000, transno 4294976611 00000004:00080000:1.0:1783520935.709262:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515347800, transno 4294976612 00000004:00080000:1.0:1783520935.709264:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515344380, transno 4294976613 00000004:00080000:1.0:1783520935.709267:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515475c00, transno 4294976614 00000004:00080000:1.0:1783520935.709269:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515345180, transno 4294976615 00000004:00080000:1.0:1783520935.709273:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cc7f100, transno 4294976616 00000004:00080000:1.0:1783520935.709275:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94361103bb80, transno 4294976617 00000004:00080000:1.0:1783520935.709278:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943611038a80, transno 4294976618 00000004:00080000:1.0:1783520935.709280:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94361103a300, transno 4294976619 00000004:00080000:1.0:1783520935.709282:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435079f0700, transno 4294976620 00000004:00080000:1.0:1783520935.709286:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435079f1180, transno 4294976621 00000004:00080000:1.0:1783520935.709291:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435079f1500, transno 4294976622 00000004:00080000:1.0:1783520935.709295:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435079f3b80, transno 4294976623 00000004:00080000:1.0:1783520935.709298:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350d164a80, transno 4294976624 00000004:00080000:1.0:1783520935.709302:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350d166d80, transno 4294976625 00000004:00080000:1.0:1783520935.709307:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435079f0000, transno 4294976626 00000004:00080000:1.0:1783520935.709312:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435079f2d80, transno 4294976627 00000004:00080000:1.0:1783520935.709315:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943505877100, transno 4294976628 00000004:00080000:1.0:1783520935.709318:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943611039500, transno 4294976629 00000004:00080000:1.0:1783520935.709321:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943505875f80, transno 4294976630 00000004:00080000:1.0:1783520935.709323:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943505875500, transno 4294976631 00000004:00080000:1.0:1783520935.709326:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c5bb100, transno 4294976632 00000004:00080000:1.0:1783520935.709328:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c5b9f80, transno 4294976633 00000004:00080000:1.0:1783520935.709330:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c5b9c00, transno 4294976634 00000004:00080000:1.0:1783520935.709335:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943638996d80, transno 4294976635 00000004:00080000:1.0:1783520935.709339:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c5b8e00, transno 4294976636 00000004:00080000:1.0:1783520935.709344:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c5b9500, transno 4294976637 00000004:00080000:1.0:1783520935.709348:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943505874700, transno 4294976638 00000004:00080000:1.0:1783520935.709353:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350794df80, transno 4294976639 00000004:00080000:1.0:1783520935.709380:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943502252680, transno 4294976640 00000004:00080000:1.0:1783520935.709386:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943502250e00, transno 4294976641 00000004:00080000:1.0:1783520935.709392:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c5b9180, transno 4294976642 00000004:00080000:1.0:1783520935.709396:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c5ba680, transno 4294976643 00000004:00080000:1.0:1783520935.709401:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b1c00, transno 4294976644 00000004:00080000:1.0:1783520935.709407:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350d165f80, transno 4294976645 00000004:00080000:1.0:1783520935.709413:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350d165180, transno 4294976646 00000004:00080000:1.0:1783520935.709418:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350d167100, transno 4294976647 00000004:00080000:1.0:1783520935.709421:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515476680, transno 4294976648 00000004:00080000:1.0:1783520935.709424:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943502250700, transno 4294976649 00000004:00080000:1.0:1783520935.709427:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515475880, transno 4294976650 00000004:00080000:1.0:1783520935.709431:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515347480, transno 4294976651 00000004:00080000:1.0:1783520935.709435:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515345500, transno 4294976652 00000004:00080000:1.0:1783520935.709440:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cc7fb80, transno 4294976653 00000004:00080000:1.0:1783520935.709446:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cc7df80, transno 4294976654 00000004:00080000:1.0:1783520935.709451:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515474e00, transno 4294976655 00000004:00080000:1.0:1783520935.709458:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515346d80, transno 4294976656 00000004:00080000:1.0:1783520935.709465:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b6680, transno 4294976657 00000004:00080000:1.0:1783520935.709470:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b6300, transno 4294976658 00000004:00080000:1.0:1783520935.709477:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b4700, transno 4294976659 00000004:00080000:1.0:1783520935.709482:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435079f1f80, transno 4294976660 00000004:00080000:1.0:1783520935.709486:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435079f1c00, transno 4294976661 00000004:00080000:1.0:1783520935.709490:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cc7c700, transno 4294976662 00000004:00080000:1.0:1783520935.709492:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435079f0380, transno 4294976663 00000004:00080000:1.0:1783520935.709494:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435079f3480, transno 4294976664 00000004:00080000:1.0:1783520935.709511:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943512426300, transno 4294976665 00000004:00080000:1.0:1783520935.709516:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513b20e00, transno 4294976666 00000004:00080000:1.0:1783520935.709520:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435079f3800, transno 4294976667 00000004:00080000:1.0:1783520935.709525:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b3b80, transno 4294976668 00000004:00080000:1.0:1783520935.709529:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350d167800, transno 4294976669 00000004:00080000:1.0:1783520935.709534:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b1500, transno 4294976670 00000004:00080000:1.0:1783520935.709538:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435079f2300, transno 4294976671 00000004:00080000:1.0:1783520935.709544:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351510b480, transno 4294976672 00000004:00080000:1.0:1783520935.709548:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350d166300, transno 4294976673 00000004:00080000:1.0:1783520935.709554:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c5ba300, transno 4294976674 00000004:00080000:1.0:1783520935.709559:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c5bbb80, transno 4294976675 00000004:00080000:1.0:1783520935.709565:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c5bb480, transno 4294976676 00000004:00080000:1.0:1783520935.709570:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350d165880, transno 4294976677 00000004:00080000:1.0:1783520935.709577:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350794ca80, transno 4294976678 00000004:00080000:1.0:1783520935.709583:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350794f480, transno 4294976679 00000004:00080000:1.0:1783520935.709589:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c5b8000, transno 4294976680 00000004:00080000:1.0:1783520935.709594:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cba2a00, transno 4294976681 00000004:00080000:1.0:1783520935.709598:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b5880, transno 4294976682 00000004:00080000:1.0:1783520935.709603:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b4a80, transno 4294976683 00000004:00080000:1.0:1783520935.709608:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cba0a80, transno 4294976684 00000004:00080000:1.0:1783520935.709613:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cba0700, transno 4294976685 00000004:00080000:1.0:1783520935.709618:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b5c00, transno 4294976686 00000004:00080000:1.0:1783520935.709622:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cba3100, transno 4294976687 00000004:00080000:1.0:1783520935.709626:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cba3480, transno 4294976688 00000004:00080000:1.0:1783520935.709632:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b7480, transno 4294976689 00000004:00080000:1.0:1783520935.709636:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b4380, transno 4294976690 00000004:00080000:1.0:1783520935.709641:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513b23100, transno 4294976691 00000004:00080000:1.0:1783520935.709645:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943509176300, transno 4294976692 00000004:00080000:1.0:1783520935.709649:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943509177100, transno 4294976693 00000004:00080000:1.0:1783520935.709655:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943502253480, transno 4294976694 00000004:00080000:1.0:1783520935.709659:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943509176680, transno 4294976695 00000004:00080000:1.0:1783520935.709663:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cba1880, transno 4294976696 00000004:00080000:1.0:1783520935.709669:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435079f0e00, transno 4294976697 00000004:00080000:1.0:1783520935.709676:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503a69500, transno 4294976698 00000004:00080000:1.0:1783520935.709683:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503a6b800, transno 4294976699 00000004:00080000:1.0:1783520935.709690:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513b05f80, transno 4294976700 00000004:00080000:1.0:1783520935.709695:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503a6bb80, transno 4294976701 00000004:00080000:1.0:1783520935.709702:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350d166680, transno 4294976702 00000004:00080000:1.0:1783520935.709708:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350d166a00, transno 4294976703 00000004:00080000:1.0:1783520935.709714:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c5baa00, transno 4294976704 00000004:00080000:1.0:1783520935.709719:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350d164700, transno 4294976705 00000004:00080000:1.0:1783520935.709723:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c5b8380, transno 4294976706 00000004:00080000:1.0:1783520935.709728:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c5b9880, transno 4294976707 00000004:00080000:1.0:1783520935.709734:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943508ee4700, transno 4294976708 00000004:00080000:1.0:1783520935.709738:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943508ee6a00, transno 4294976709 00000004:00080000:1.0:1783520935.709742:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943508ee4000, transno 4294976710 00000004:00080000:1.0:1783520935.709747:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513b20a80, transno 4294976711 00000004:00080000:1.0:1783520935.709752:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943508ee5c00, transno 4294976712 00000004:00080000:1.0:1783520935.709756:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513b22300, transno 4294976713 00000004:00080000:1.0:1783520935.709761:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513b22680, transno 4294976714 00000004:00080000:1.0:1783520935.709765:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513b21880, transno 4294976715 00000004:00080000:1.0:1783520935.709771:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513b20000, transno 4294976716 00000004:00080000:1.0:1783520935.709776:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943502250380, transno 4294976717 00000004:00080000:1.0:1783520935.709780:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513b22a00, transno 4294976718 00000004:00080000:1.0:1783520935.709784:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513b23480, transno 4294976719 00000004:00080000:1.0:1783520935.709788:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513b21f80, transno 4294976720 00000004:00080000:1.0:1783520935.709792:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943502250000, transno 4294976721 00000004:00080000:1.0:1783520935.709797:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943508ee4380, transno 4294976722 00000004:00080000:1.0:1783520935.709802:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503a69880, transno 4294976723 00000004:00080000:1.0:1783520935.709805:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943517330380, transno 4294976724 00000004:00080000:1.0:1783520935.709810:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515344700, transno 4294976725 00000004:00080000:1.0:1783520935.709816:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cba0000, transno 4294976726 00000004:00080000:1.0:1783520935.709821:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513bc3800, transno 4294976727 00000004:00080000:1.0:1783520935.709825:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cba1500, transno 4294976728 00000004:00080000:1.0:1783520935.709829:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cba2300, transno 4294976729 00000004:00080000:1.0:1783520935.709833:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503a6ad80, transno 4294976730 00000004:00080000:1.0:1783520935.709837:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503a6b480, transno 4294976731 00000004:00080000:1.0:1783520935.709840:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513bc0a80, transno 4294976732 00000004:00080000:1.0:1783520935.709844:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503a6a300, transno 4294976733 00000004:00080000:1.0:1783520935.709847:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513bc1880, transno 4294976734 00000004:00080000:1.0:1783520935.709851:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b2300, transno 4294976735 00000004:00080000:1.0:1783520935.709856:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cba0380, transno 4294976736 00000004:00080000:1.0:1783520935.709861:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435032eb480, transno 4294976737 00000004:00080000:1.0:1783520935.709867:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435032ea300, transno 4294976738 00000004:00080000:1.0:1783520935.709873:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b2a00, transno 4294976739 00000004:00080000:1.0:1783520935.709879:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513bc2d80, transno 4294976740 00000004:00080000:1.0:1783520935.709885:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943508ee6300, transno 4294976741 00000004:00080000:1.0:1783520935.709892:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e3b2680, transno 4294976742 00000004:00080000:1.0:1783520935.709898:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e3b0a80, transno 4294976743 00000004:00080000:1.0:1783520935.709904:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e3b3b80, transno 4294976744 00000004:00080000:1.0:1783520935.709910:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943517333800, transno 4294976745 00000004:00080000:1.0:1783520935.709915:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b1f80, transno 4294976746 00000004:00080000:1.0:1783520935.709920:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943517333480, transno 4294976747 00000004:00080000:1.0:1783520935.709927:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943517331880, transno 4294976748 00000004:00080000:1.0:1783520935.709932:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943517332a00, transno 4294976749 00000004:00080000:1.0:1783520935.709938:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513b23b80, transno 4294976750 00000004:00080000:1.0:1783520935.709944:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b0000, transno 4294976751 00000004:00080000:1.0:1783520935.709949:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b1180, transno 4294976752 00000004:00080000:1.0:1783520935.709953:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cba3b80, transno 4294976753 00000004:00080000:1.0:1783520935.709956:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cba2680, transno 4294976754 00000004:00080000:1.0:1783520935.709959:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515345f80, transno 4294976755 00000004:00080000:1.0:1783520935.709963:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943508ee4e00, transno 4294976756 00000004:00080000:1.0:1783520935.709967:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cba1f80, transno 4294976757 00000004:00080000:1.0:1783520935.709972:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513b20380, transno 4294976758 00000004:00080000:1.0:1783520935.709975:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513b22d80, transno 4294976759 00000004:00080000:1.0:1783520935.709979:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cc7ca80, transno 4294976760 00000004:00080000:1.0:1783520935.709983:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943517331f80, transno 4294976761 00000004:00080000:1.0:1783520935.709988:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b7b80, transno 4294976762 00000004:00080000:1.0:1783520935.709992:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943517333b80, transno 4294976763 00000004:00080000:1.0:1783520935.709997:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94361103b100, transno 4294976764 00000004:00080000:1.0:1783520935.710002:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b5180, transno 4294976765 00000004:00080000:1.0:1783520935.710007:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943508ee6d80, transno 4294976766 00000004:00080000:1.0:1783520935.710013:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943508ee4a80, transno 4294976767 00000004:00080000:1.0:1783520935.710018:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94361103b480, transno 4294976768 00000004:00080000:1.0:1783520935.710024:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513138000, transno 4294976769 00000004:00080000:1.0:1783520935.710032:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943509a80700, transno 4294976770 00000001:00080000:3.0:1783520935.710039:0:9485:0:(tgt_lastrcvd.c:983:tgt_cb_last_committed()) lustre-OST0001: transno 4294975574 is committed 00000004:00080000:1.0:1783520935.710039:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943511ad9880, transno 4294976771 00000004:00080000:1.0:1783520935.710047:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435032eaa00, transno 4294976772 00000001:00080000:3.0:1783520935.710049:0:9485:0:(tgt_lastrcvd.c:983:tgt_cb_last_committed()) lustre-OST0001: transno 4294975573 is committed 00000004:00080000:1.0:1783520935.710053:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943509a81880, transno 4294976773 00000004:00080000:1.0:1783520935.710059:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943638995500, transno 4294976774 00000004:00080000:1.0:1783520935.710065:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943509a82a00, transno 4294976775 00000004:00080000:1.0:1783520935.710068:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943511ada680, transno 4294976776 00000004:00080000:1.0:1783520935.710071:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513139500, transno 4294976777 00000004:00080000:1.0:1783520935.710077:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cc7ea00, transno 4294976778 00000004:00080000:1.0:1783520935.710080:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513bc3480, transno 4294976779 00000004:00080000:1.0:1783520935.710084:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503a6b100, transno 4294976780 00000004:00080000:1.0:1783520935.710089:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943505876300, transno 4294976781 00000004:00080000:1.0:1783520935.710093:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943508ee5180, transno 4294976782 00000004:00080000:1.0:1783520935.710098:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943512427800, transno 4294976783 00000100:00100000:3.0:1783520935.710101:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002363008 00000004:00080000:1.0:1783520935.710104:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943508ee7100, transno 4294976784 00000100:00100000:3.0:1783520935.710106:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002363008 00000004:00080000:1.0:1783520935.710110:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e11ea00, transno 4294976785 00000004:00080000:1.0:1783520935.710116:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e11e680, transno 4294976786 00000004:00080000:1.0:1783520935.710121:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514dd5c00, transno 4294976787 00010000:00010000:3.0:1783520935.710125:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000040f83ede/0xd5401cbabc01f203 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x527:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef908 expref: 981 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.710127:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e11c700, transno 4294976788 00000004:00080000:1.0:1783520935.710134:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943512427100, transno 4294976789 00000004:00080000:1.0:1783520935.710142:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e11df80, transno 4294976790 00000004:00080000:1.0:1783520935.710147:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943512426680, transno 4294976791 00000100:00100000:3.0:1783520935.710150:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35675 00000004:00080000:1.0:1783520935.710154:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e11d180, transno 4294976792 00000100:00100000:3.0:1783520935.710159:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517246d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+981:2388:x1870153002363008:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000004:00080000:1.0:1783520935.710160:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e11ed80, transno 4294976793 00010000:00010000:3.0:1783520935.710166:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000004:00080000:1.0:1783520935.710167:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350ade3480, transno 4294976794 00010000:00010000:3.0:1783520935.710174:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000040f83ede/0xd5401cbabc01f203 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x527:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef908 expref: 981 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.710174:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350ade0a80, transno 4294976795 00000004:00080000:1.0:1783520935.710181:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943511adbb80, transno 4294976796 00000004:00080000:1.0:1783520935.710187:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943511adb100, transno 4294976797 00010000:00010000:3.0:1783520935.710188:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000040f83ede/0xd5401cbabc01f203 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x527:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef908 expref: 981 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.710194:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b5500, transno 4294976798 00000004:00080000:1.0:1783520935.710199:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350ade3100, transno 4294976799 00010000:00010000:3.0:1783520935.710202:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000040f83ede/0xd5401cbabc01f203 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x527:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef908 expref: 981 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.710205:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943511ad9180, transno 4294976800 00000004:00080000:1.0:1783520935.710212:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943511ad9500, transno 4294976801 00010000:00010000:3.0:1783520935.710218:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x527:0x0].0x0 since=0s v=(0 0) 00000004:00080000:1.0:1783520935.710219:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514dd6d80, transno 4294976802 00010000:00010000:3.0:1783520935.710225:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x527:0x0].0x0 since=0s v=(0 0) 00000004:00080000:1.0:1783520935.710226:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b4000, transno 4294976803 00010000:00010000:3.0:1783520935.710229:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000013dcb818/0xd5401cbabc02509f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x527:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.710232:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350ade1c00, transno 4294976804 00010000:00010000:3.0:1783520935.710239:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000013dcb818/0xd5401cbabc02509f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x527:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.710240:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943509a82d80, transno 4294976805 00000004:00080000:1.0:1783520935.710246:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350ade0e00, transno 4294976806 00000004:00080000:1.0:1783520935.710251:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514dd5f80, transno 4294976807 00000004:00080000:1.0:1783520935.710257:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351313b100, transno 4294976808 00000004:00080000:1.0:1783520935.710261:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350ade3b80, transno 4294976809 00000004:00080000:1.0:1783520935.710265:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514dd4000, transno 4294976810 00010000:00010000:3.0:1783520935.710267:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000004:00080000:1.0:1783520935.710270:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350ade1880, transno 4294976811 00000004:00080000:1.0:1783520935.710274:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943511ad8700, transno 4294976812 00000004:00080000:1.0:1783520935.710278:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943638996680, transno 4294976813 00000004:00080000:1.0:1783520935.710283:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943505876680, transno 4294976814 00000004:00080000:1.0:1783520935.710289:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943638997800, transno 4294976815 00000100:00100000:3.0:1783520935.710295:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517246d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+980:2388:x1870153002363008:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 136us (1632us total) trans 0 rc 0/0 00000004:00080000:1.0:1783520935.710298:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943638994000, transno 4294976816 00000100:00100000:3.0:1783520935.710304:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35675 00000004:00080000:1.0:1783520935.710305:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943638994a80, transno 4294976817 00000004:00080000:1.0:1783520935.710312:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943638994e00, transno 4294976818 00000004:00080000:1.0:1783520935.710316:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943638994700, transno 4294976819 00000004:00080000:1.0:1783520935.710322:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943509a83b80, transno 4294976820 00000004:00080000:1.0:1783520935.710327:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e3b1880, transno 4294976821 00000004:00080000:1.0:1783520935.710333:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943502253800, transno 4294976822 00000004:00080000:1.0:1783520935.710338:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943509a81500, transno 4294976823 00000004:00080000:1.0:1783520935.710343:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943502252a00, transno 4294976824 00000004:00080000:1.0:1783520935.710347:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515345880, transno 4294976825 00000004:00080000:1.0:1783520935.710353:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943502253b80, transno 4294976826 00000004:00080000:1.0:1783520935.710394:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943502253100, transno 4294976827 00010000:00010000:3.0:1783520935.710399:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000004:00080000:1.0:1783520935.710400:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c5bb800, transno 4294976828 00010000:00010000:3.0:1783520935.710408:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000013dcb818/0xd5401cbabc02509f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x527:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.710408:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943517332680, transno 4294976829 00000004:00080000:1.0:1783520935.710413:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943509174e00, transno 4294976830 00000004:00080000:1.0:1783520935.710418:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943517332d80, transno 4294976831 00010000:00010000:3.0:1783520935.710425:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf7e00 already unused, calling callback (ffffffffc0bf9e90) 00000004:00080000:1.0:1783520935.710426:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943509174700, transno 4294976832 00010000:00010000:3.0:1783520935.710432:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000013dcb818/0xd5401cbabc02509f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x527:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.710433:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503325f80, transno 4294976833 00000004:00080000:1.0:1783520935.710439:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514dd5500, transno 4294976834 00000004:00080000:1.0:1783520935.710444:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514dd4e00, transno 4294976835 00010000:00010000:3.0:1783520935.710448:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000013dcb818/0xd5401cbabc02509f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x527:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.710449:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350ade1500, transno 4294976836 00000004:00080000:1.0:1783520935.710455:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514dd6a00, transno 4294976837 00000004:00080000:1.0:1783520935.710463:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943634572300, transno 4294976838 00000004:00080000:1.0:1783520935.710467:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503326a00, transno 4294976839 00010000:00010000:3.0:1783520935.710468:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000013dcb818/0xd5401cbabc02509f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x527:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.710472:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350ade0700, transno 4294976840 00000004:00080000:1.0:1783520935.710477:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514dd4700, transno 4294976841 00000004:00080000:1.0:1783520935.710482:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503325880, transno 4294976842 00000004:00080000:1.0:1783520935.710488:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503324700, transno 4294976843 00010000:00010000:3.0:1783520935.710490:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000013dcb818/0xd5401cbabc02509f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x527:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.710493:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503326680, transno 4294976844 00010000:00010000:3.0:1783520935.710526:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.710529:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.710532:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520935.710538:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515776a00, transno 4294976845 00000004:00080000:1.0:1783520935.710543:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514dd7100, transno 4294976846 00000004:00080000:1.0:1783520935.710549:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515776d80, transno 4294976847 00000004:00080000:1.0:1783520935.710552:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515777100, transno 4294976848 00000004:00080000:1.0:1783520935.710555:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943638996a00, transno 4294976849 00000004:00080000:1.0:1783520935.710558:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943611039f80, transno 4294976850 00000004:00080000:1.0:1783520935.710562:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513bc3b80, transno 4294976851 00000004:00080000:1.0:1783520935.710565:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503326300, transno 4294976852 00000004:00080000:1.0:1783520935.710569:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943508ee7b80, transno 4294976853 00000004:00080000:1.0:1783520935.710573:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513bc2a00, transno 4294976854 00000004:00080000:1.0:1783520935.710578:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503371500, transno 4294976855 00000004:00080000:1.0:1783520935.710584:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513bc0000, transno 4294976856 00000004:00080000:1.0:1783520935.710589:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943512427480, transno 4294976857 00000004:00080000:1.0:1783520935.710594:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e11f100, transno 4294976858 00000004:00080000:1.0:1783520935.710600:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503327800, transno 4294976859 00000004:00080000:1.0:1783520935.710605:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503371f80, transno 4294976860 00000004:00080000:1.0:1783520935.710611:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503372300, transno 4294976861 00000004:00080000:1.0:1783520935.710614:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e11ce00, transno 4294976862 00000004:00080000:1.0:1783520935.710618:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503371880, transno 4294976863 00000004:00080000:1.0:1783520935.710624:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e11c380, transno 4294976864 00000004:00080000:1.0:1783520935.710628:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350626ca80, transno 4294976865 00000004:00080000:1.0:1783520935.710634:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943502252300, transno 4294976866 00000004:00080000:1.0:1783520935.710638:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503371180, transno 4294976867 00000004:00080000:1.0:1783520935.710643:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503371c00, transno 4294976868 00000004:00080000:1.0:1783520935.710647:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350626c700, transno 4294976869 00000004:00080000:1.0:1783520935.710651:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363d3a4e00, transno 4294976870 00000004:00080000:1.0:1783520935.710656:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363d3a5500, transno 4294976871 00000004:00080000:1.0:1783520935.710661:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363d3a4700, transno 4294976872 00000004:00080000:1.0:1783520935.710665:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503327480, transno 4294976873 00000004:00080000:1.0:1783520935.710670:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363d3a5180, transno 4294976874 00000004:00080000:1.0:1783520935.710674:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503327100, transno 4294976875 00000004:00080000:1.0:1783520935.710677:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350626d500, transno 4294976876 00000004:00080000:1.0:1783520935.710680:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514dd6300, transno 4294976877 00000004:00080000:1.0:1783520935.710683:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350626df80, transno 4294976878 00000004:00080000:1.0:1783520935.710686:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350626ed80, transno 4294976879 00000004:00080000:1.0:1783520935.710689:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943509175500, transno 4294976880 00000004:00080000:1.0:1783520935.710693:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943611038700, transno 4294976881 00000004:00080000:1.0:1783520935.710697:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503370e00, transno 4294976882 00000004:00080000:1.0:1783520935.710701:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350626ce00, transno 4294976883 00000004:00080000:1.0:1783520935.710704:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363fe54e00, transno 4294976884 00000004:00080000:1.0:1783520935.710710:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435166c8a80, transno 4294976885 00000004:00080000:1.0:1783520935.710714:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350626d180, transno 4294976886 00000004:00080000:1.0:1783520935.710717:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515775500, transno 4294976887 00000004:00080000:1.0:1783520935.710721:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514dd7800, transno 4294976888 00000004:00080000:1.0:1783520935.710726:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363fe55500, transno 4294976889 00000004:00080000:1.0:1783520935.710730:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363fe55c00, transno 4294976890 00000004:00080000:1.0:1783520935.710735:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514dd5180, transno 4294976891 00000004:00080000:1.0:1783520935.710742:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351720b100, transno 4294976892 00000004:00080000:1.0:1783520935.710748:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435166c9500, transno 4294976893 00000004:00080000:1.0:1783520935.710754:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351720b480, transno 4294976894 00000004:00080000:1.0:1783520935.710759:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516649c00, transno 4294976895 00000004:00080000:1.0:1783520935.710765:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516648380, transno 4294976896 00000004:00080000:1.0:1783520935.710768:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516649880, transno 4294976897 00000004:00080000:1.0:1783520935.710770:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94360550e300, transno 4294976898 00000004:00080000:1.0:1783520935.710774:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351664bb80, transno 4294976899 00000004:00080000:1.0:1783520935.710777:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94360550f800, transno 4294976900 00000004:00080000:1.0:1783520935.710780:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94360550ed80, transno 4294976901 00000004:00080000:1.0:1783520935.710783:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94360550e680, transno 4294976902 00000004:00080000:1.0:1783520935.710786:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94360550c000, transno 4294976903 00010000:00010000:3.0:1783520935.710789:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000004:00080000:1.0:1783520935.710790:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e0f7b80, transno 4294976904 00010000:00010000:3.0:1783520935.710795:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025060 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x520:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.710797:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94360550d880, transno 4294976905 00000004:00080000:1.0:1783520935.710803:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943509a80380, transno 4294976906 00010000:00010000:3.0:1783520935.710805:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.710811:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025060 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x520:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.710811:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943509a82680, transno 4294976907 00000004:00080000:1.0:1783520935.710817:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943509a80e00, transno 4294976908 00010000:00010000:3.0:1783520935.710821:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025060 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x520:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.710823:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351635a300, transno 4294976909 00000004:00080000:1.0:1783520935.710830:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351635b480, transno 4294976910 00010000:00010000:3.0:1783520935.710836:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025060 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x520:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.710836:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351635ad80, transno 4294976911 00000004:00080000:1.0:1783520935.710842:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516358a80, transno 4294976912 00010000:00010000:3.0:1783520935.710846:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025060 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x520:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.710848:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943509a80000, transno 4294976913 00000004:00080000:1.0:1783520935.710855:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516359500, transno 4294976914 00000004:00080000:1.0:1783520935.710860:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516359880, transno 4294976915 00000004:00080000:1.0:1783520935.710868:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503373480, transno 4294976916 00010000:00010000:3.0:1783520935.710870:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.710874:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520935.710874:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351635aa00, transno 4294976917 00010000:00010000:3.0:1783520935.710876:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520935.710880:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350ade2d80, transno 4294976918 00000004:00080000:1.0:1783520935.710885:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363d3a7b80, transno 4294976919 00000004:00080000:1.0:1783520935.710888:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943634571c00, transno 4294976920 00000004:00080000:1.0:1783520935.710891:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943511ad9f80, transno 4294976921 00000004:00080000:1.0:1783520935.710895:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943511ad9c00, transno 4294976922 00000004:00080000:1.0:1783520935.710898:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943511adad80, transno 4294976923 00000004:00080000:1.0:1783520935.710902:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513b20700, transno 4294976924 00000004:00080000:1.0:1783520935.710905:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435161c2a00, transno 4294976925 00000004:00080000:1.0:1783520935.710909:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e3b0700, transno 4294976926 00000004:00080000:1.0:1783520935.710913:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435038eea00, transno 4294976927 00000004:00080000:1.0:1783520935.710917:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435038eca80, transno 4294976928 00000004:00080000:1.0:1783520935.710923:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943511adb480, transno 4294976929 00000004:00080000:1.0:1783520935.710929:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435038ece00, transno 4294976930 00000004:00080000:1.0:1783520935.710932:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350626dc00, transno 4294976931 00000004:00080000:1.0:1783520935.710936:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435080d1180, transno 4294976932 00000004:00080000:1.0:1783520935.710940:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435080d2a00, transno 4294976933 00000004:00080000:1.0:1783520935.710944:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350626c380, transno 4294976934 00000004:00080000:1.0:1783520935.710947:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350626e680, transno 4294976935 00000004:00080000:1.0:1783520935.710952:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435161c2d80, transno 4294976936 00000004:00080000:1.0:1783520935.710956:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435161c3100, transno 4294976937 00000004:00080000:1.0:1783520935.710961:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943508ee5f80, transno 4294976938 00000004:00080000:1.0:1783520935.710965:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e11dc00, transno 4294976939 00000004:00080000:1.0:1783520935.710971:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503327b80, transno 4294976940 00000004:00080000:1.0:1783520935.710974:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943509174a80, transno 4294976941 00000004:00080000:1.0:1783520935.710978:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503326d80, transno 4294976942 00000004:00080000:1.0:1783520935.710981:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503a68a80, transno 4294976943 00000004:00080000:1.0:1783520935.710985:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9436342b8a80, transno 4294976944 00000004:00080000:1.0:1783520935.710987:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9436342b9880, transno 4294976945 00000004:00080000:1.0:1783520935.710991:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9436342b8000, transno 4294976946 00000004:00080000:1.0:1783520935.710995:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9436342b9500, transno 4294976947 00000004:00080000:1.0:1783520935.710998:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9436342ba300, transno 4294976948 00000004:00080000:1.0:1783520935.711004:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9436342bb480, transno 4294976949 00000004:00080000:1.0:1783520935.711007:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9436342b9180, transno 4294976950 00000004:00080000:1.0:1783520935.711012:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503324000, transno 4294976951 00000004:00080000:1.0:1783520935.711014:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435080d2d80, transno 4294976952 00000004:00080000:1.0:1783520935.711017:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351dc42a00, transno 4294976953 00000004:00080000:1.0:1783520935.711019:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351dc41180, transno 4294976954 00000004:00080000:1.0:1783520935.711023:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351dc43480, transno 4294976955 00000004:00080000:1.0:1783520935.711026:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514ff2a00, transno 4294976956 00000004:00080000:1.0:1783520935.711029:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514ff1f80, transno 4294976957 00000004:00080000:1.0:1783520935.711031:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514ff1c00, transno 4294976958 00000004:00080000:1.0:1783520935.711033:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435080d1f80, transno 4294976959 00000004:00080000:1.0:1783520935.711037:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514ff2680, transno 4294976960 00000004:00080000:1.0:1783520935.711041:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514ff2300, transno 4294976961 00000004:00080000:1.0:1783520935.711045:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435080d3100, transno 4294976962 00000004:00080000:1.0:1783520935.711049:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514d28a80, transno 4294976963 00000004:00080000:1.0:1783520935.711054:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514d2a300, transno 4294976964 00000100:00100000:3.0:1783520935.711056:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002363136 00000004:00080000:1.0:1783520935.711058:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514d2b100, transno 4294976965 00000100:00100000:3.0:1783520935.711063:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002363136 00000004:00080000:1.0:1783520935.711064:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514dd5880, transno 4294976966 00000004:00080000:1.0:1783520935.711069:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435166c8000, transno 4294976967 00000004:00080000:1.0:1783520935.711073:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514d28000, transno 4294976968 00000004:00080000:1.0:1783520935.711075:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514d2a680, transno 4294976969 00000004:00080000:1.0:1783520935.711079:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514d2b480, transno 4294976970 00000004:00080000:1.0:1783520935.711082:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514d29c00, transno 4294976971 00000004:00080000:1.0:1783520935.711085:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514d2b800, transno 4294976972 00000004:00080000:1.0:1783520935.711088:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516310700, transno 4294976973 00000004:00080000:1.0:1783520935.711092:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516311500, transno 4294976974 00000004:00080000:1.0:1783520935.711095:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516311880, transno 4294976975 00000100:00100000:3.0:1783520935.711098:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 44084 00000004:00080000:1.0:1783520935.711103:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516313800, transno 4294976976 00000004:00080000:1.0:1783520935.711108:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516310e00, transno 4294976977 00000100:00100000:3.0:1783520935.711110:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507acf100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+980:251787:x1870153002363136:12345-192.168.204.33@tcp:101:dd.0 00000004:00080000:1.0:1783520935.711113:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516312a00, transno 4294976978 00000004:00080000:1.0:1783520935.711118:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9436342b9f80, transno 4294976979 00010000:00010000:3.0:1783520935.711119:0:12734:0:(ldlm_lockd.c:1265:ldlm_handle_enqueue()) ### server-side enqueue handler START 00000004:00080000:1.0:1783520935.711121:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351dc42680, transno 4294976980 00000004:00080000:1.0:1783520935.711126:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503325c00, transno 4294976981 00000004:00080000:1.0:1783520935.711130:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9436342b8700, transno 4294976982 00010000:00010000:3.0:1783520935.711134:0:12734:0:(ldlm_lockd.c:1343:ldlm_handle_enqueue()) ### server-side enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0250bb lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0xc275876b859f34a4 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.711136:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514d29180, transno 4294976983 00000004:00080000:1.0:1783520935.711143:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514d2bb80, transno 4294976984 00000004:00080000:1.0:1783520935.711150:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514d28e00, transno 4294976985 00000004:00080000:1.0:1783520935.711154:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943508ee5500, transno 4294976986 00000004:00080000:1.0:1783520935.711157:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514dd6680, transno 4294976987 00000004:00080000:1.0:1783520935.711161:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514ff2d80, transno 4294976988 00000004:00080000:1.0:1783520935.711164:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351dc42d80, transno 4294976989 00000004:00080000:1.0:1783520935.711168:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514ff3b80, transno 4294976990 00000004:00080000:1.0:1783520935.711171:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350544f100, transno 4294976991 00000004:00080000:1.0:1783520935.711174:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435038ee680, transno 4294976992 00000004:00080000:1.0:1783520935.711179:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943638995180, transno 4294976993 00000004:00080000:1.0:1783520935.711185:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9436342b9c00, transno 4294976994 00000004:00080000:1.0:1783520935.711189:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350626ea00, transno 4294976995 00000004:00080000:1.0:1783520935.711193:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435032f1880, transno 4294976996 00000004:00080000:1.0:1783520935.711199:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435032e8000, transno 4294976997 00000004:00080000:1.0:1783520935.711206:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435161c0e00, transno 4294976998 00000004:00080000:1.0:1783520935.711211:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435032f0000, transno 4294976999 00000004:00080000:1.0:1783520935.711216:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435038ef100, transno 4294977000 00000004:00080000:1.0:1783520935.711221:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e11c000, transno 4294977001 00002000:00010000:3.0:1783520935.711225:0:12734:0:(ofd_lvb.c:151:ofd_lvbo_init()) res: [0x2c0000401:0x8f7:0x0] initial LVB size: 0, mtime: 0x0, atime: 0x0, ctime: 0x0, blocks: 0x0 00000004:00080000:1.0:1783520935.711225:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943509176a00, transno 4294977002 00000004:00080000:1.0:1783520935.711230:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513138380, transno 4294977003 00000004:00080000:1.0:1783520935.711235:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e0f5c00, transno 4294977004 00000004:00080000:1.0:1783520935.711239:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e11e300, transno 4294977005 00010000:00010000:3.0:1783520935.711241:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f7:0x0].0x0 since=4130s v=(0 0) 00000004:00080000:1.0:1783520935.711243:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e11ca80, transno 4294977006 00010000:00010000:3.0:1783520935.711249:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f7:0x0].0x0 since=0s v=(0 0) 00000004:00080000:1.0:1783520935.711251:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516359c00, transno 4294977007 00000004:00080000:1.0:1783520935.711256:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516358700, transno 4294977008 00010000:00010000:3.0:1783520935.711259:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0250bb lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34a4 expref: 981 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.711261:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503370380, transno 4294977009 00000004:00080000:1.0:1783520935.711267:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351635b800, transno 4294977010 00000004:00080000:1.0:1783520935.711273:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516359180, transno 4294977011 00000004:00080000:1.0:1783520935.711279:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b1880, transno 4294977012 00010000:00010000:3.0:1783520935.711281:0:12734:0:(ldlm_lockd.c:1513:ldlm_handle_enqueue()) ### server-side enqueue handler, sending reply (err=0, rc=0) ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0250bb lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34a4 expref: 982 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.711285:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b0a80, transno 4294977013 00000004:00080000:1.0:1783520935.711291:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e007b80, transno 4294977014 00000004:00080000:1.0:1783520935.711297:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e004e00, transno 4294977015 00000004:00080000:1.0:1783520935.711303:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514ff0e00, transno 4294977016 00010000:00010000:3.0:1783520935.711306:0:12734:0:(ldlm_lockd.c:1592:ldlm_handle_enqueue()) ### server-side enqueue handler END (lock ffff94351cbf4400, rc 0) 00000004:00080000:1.0:1783520935.711309:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516681880, transno 4294977017 00000004:00080000:1.0:1783520935.711314:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351575e300, transno 4294977018 00000004:00080000:1.0:1783520935.711319:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351575f800, transno 4294977019 00000004:00080000:1.0:1783520935.711325:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435080d1500, transno 4294977020 00000004:00080000:1.0:1783520935.711330:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435080d3480, transno 4294977021 00000004:00080000:1.0:1783520935.711336:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516649f80, transno 4294977022 00000100:00100000:3.0:1783520935.711338:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507acf100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+982:251787:x1870153002363136:12345-192.168.204.33@tcp:101:dd.0 Request processed in 231us (335us total) trans 0 rc 0/0 00000004:00080000:1.0:1783520935.711340:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435080d0380, transno 4294977023 00000004:00080000:1.0:1783520935.711346:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363fe56680, transno 4294977024 00000100:00100000:3.0:1783520935.711352:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 44084 00000004:00080000:1.0:1783520935.711352:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363fe56d80, transno 4294977025 00000004:00080000:1.0:1783520935.711377:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943517209c00, transno 4294977026 00000004:00080000:1.0:1783520935.711382:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94360550df80, transno 4294977027 00000004:00080000:1.0:1783520935.711387:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cba8380, transno 4294977028 00000004:00080000:1.0:1783520935.711392:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943509a83800, transno 4294977029 00000004:00080000:1.0:1783520935.711397:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943509a83100, transno 4294977030 00000004:00080000:1.0:1783520935.711401:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943502c98380, transno 4294977031 00000004:00080000:1.0:1783520935.711405:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943502c99180, transno 4294977032 00000004:00080000:1.0:1783520935.711409:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943509a81c00, transno 4294977033 00000004:00080000:1.0:1783520935.711412:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363840f100, transno 4294977034 00000004:00080000:1.0:1783520935.711417:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94360550dc00, transno 4294977035 00000004:00080000:1.0:1783520935.711422:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943634573800, transno 4294977036 00000004:00080000:1.0:1783520935.711427:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363840ed80, transno 4294977037 00000004:00080000:1.0:1783520935.711432:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943502c98e00, transno 4294977038 00000004:00080000:1.0:1783520935.711437:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e0f7480, transno 4294977039 00000004:00080000:1.0:1783520935.711441:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943502c99500, transno 4294977040 00000004:00080000:1.0:1783520935.711446:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943502c98000, transno 4294977041 00000004:00080000:1.0:1783520935.711450:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351635a680, transno 4294977042 00000004:00080000:1.0:1783520935.711455:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363840df80, transno 4294977043 00000004:00080000:1.0:1783520935.711460:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351635b100, transno 4294977044 00000004:00080000:1.0:1783520935.711466:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351664aa00, transno 4294977045 00000004:00080000:1.0:1783520935.711470:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513bc1f80, transno 4294977046 00000004:00080000:1.0:1783520935.711476:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94360550ce00, transno 4294977047 00000004:00080000:1.0:1783520935.711480:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516682300, transno 4294977048 00000004:00080000:1.0:1783520935.711485:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943502c99c00, transno 4294977049 00000004:00080000:1.0:1783520935.711489:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514ff1180, transno 4294977050 00000004:00080000:1.0:1783520935.711494:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350ade1f80, transno 4294977051 00000004:00080000:1.0:1783520935.711511:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363fe57100, transno 4294977052 00000004:00080000:1.0:1783520935.711516:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943502c9aa00, transno 4294977053 00000004:00080000:1.0:1783520935.711519:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516681180, transno 4294977054 00000004:00080000:1.0:1783520935.711522:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516313480, transno 4294977055 00000004:00080000:1.0:1783520935.711525:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9436342bb800, transno 4294977056 00000004:00080000:1.0:1783520935.711528:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516310a80, transno 4294977057 00000004:00080000:1.0:1783520935.711534:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516310000, transno 4294977058 00000004:00080000:1.0:1783520935.711537:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516311c00, transno 4294977059 00000004:00080000:1.0:1783520935.711539:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351dc41f80, transno 4294977060 00000004:00080000:1.0:1783520935.711542:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516311f80, transno 4294977061 00000004:00080000:1.0:1783520935.711546:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435161c3b80, transno 4294977062 00010000:00010000:3.0:1783520935.711553:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc0250ad lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x529:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.711553:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943512424a80, transno 4294977063 00000004:00080000:1.0:1783520935.711559:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943513b21500, transno 4294977064 00000004:00080000:1.0:1783520935.711566:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350626f800, transno 4294977065 00000004:00080000:1.0:1783520935.711574:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9436342baa00, transno 4294977066 00000004:00080000:1.0:1783520935.711578:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514d29f80, transno 4294977067 00010000:00010000:3.0:1783520935.711582:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc0250ad lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x529:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.711582:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514d2ad80, transno 4294977068 00000004:00080000:1.0:1783520935.711589:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514d29500, transno 4294977069 00000004:00080000:1.0:1783520935.711594:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363840e300, transno 4294977070 00010000:00010000:3.0:1783520935.711597:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc0250ad lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x529:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.711600:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435038ed500, transno 4294977071 00000004:00080000:1.0:1783520935.711606:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435038ee300, transno 4294977072 00010000:00010000:3.0:1783520935.711612:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc0250ad lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x529:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.711613:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130f6d80, transno 4294977073 00000004:00080000:1.0:1783520935.711620:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e11f800, transno 4294977074 00000004:00080000:1.0:1783520935.711626:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943509a81f80, transno 4294977075 00000004:00080000:1.0:1783520935.711633:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435038eed80, transno 4294977076 00000004:00080000:1.0:1783520935.711640:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943502c9b480, transno 4294977077 00000004:00080000:1.0:1783520935.711644:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943502c9a680, transno 4294977078 00000004:00080000:1.0:1783520935.711647:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943502c98a80, transno 4294977079 00010000:00010000:3.0:1783520935.711648:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000004:00080000:1.0:1783520935.711652:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943502c99880, transno 4294977080 00000004:00080000:1.0:1783520935.711658:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435080d1c00, transno 4294977081 00000004:00080000:1.0:1783520935.711661:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e007480, transno 4294977082 00000004:00080000:1.0:1783520935.711666:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943502c9b800, transno 4294977083 00000004:00080000:1.0:1783520935.711671:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435080d2680, transno 4294977084 00000004:00080000:1.0:1783520935.711675:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943511ad8000, transno 4294977085 00000004:00080000:1.0:1783520935.711681:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514ff0000, transno 4294977086 00000004:00080000:1.0:1783520935.711686:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130f6a00, transno 4294977087 00000004:00080000:1.0:1783520935.711691:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516313b80, transno 4294977088 00000004:00080000:1.0:1783520935.711695:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943508058700, transno 4294977089 00000004:00080000:1.0:1783520935.711702:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514ff1880, transno 4294977090 00000004:00080000:1.0:1783520935.711706:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350805b800, transno 4294977091 00000004:00080000:1.0:1783520935.711710:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943508058380, transno 4294977092 00000004:00080000:1.0:1783520935.711715:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350805bb80, transno 4294977093 00000004:00080000:1.0:1783520935.711720:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350805ad80, transno 4294977094 00000004:00080000:1.0:1783520935.711724:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350805b100, transno 4294977095 00000004:00080000:1.0:1783520935.711730:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350805b480, transno 4294977096 00000004:00080000:1.0:1783520935.711735:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943508058a80, transno 4294977097 00000004:00080000:1.0:1783520935.711739:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943508059f80, transno 4294977098 00000004:00080000:1.0:1783520935.711744:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943508059500, transno 4294977099 00000004:00080000:1.0:1783520935.711748:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350805a680, transno 4294977100 00000004:00080000:1.0:1783520935.711752:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363840c380, transno 4294977101 00000004:00080000:1.0:1783520935.711757:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130e0a80, transno 4294977102 00000004:00080000:1.0:1783520935.711761:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363840e680, transno 4294977103 00000004:00080000:1.0:1783520935.711766:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943511ad8a80, transno 4294977104 00000004:00080000:1.0:1783520935.711773:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130e0e00, transno 4294977105 00000004:00080000:1.0:1783520935.711779:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943508058e00, transno 4294977106 00000004:00080000:1.0:1783520935.711782:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130e1c00, transno 4294977107 00000004:00080000:1.0:1783520935.711786:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130e1180, transno 4294977108 00000004:00080000:1.0:1783520935.711789:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130e3800, transno 4294977109 00000004:00080000:1.0:1783520935.711793:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435151a7100, transno 4294977110 00000004:00080000:1.0:1783520935.711797:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435151a4a80, transno 4294977111 00000004:00080000:1.0:1783520935.711801:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130e3480, transno 4294977112 00000004:00080000:1.0:1783520935.711805:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435151a6a00, transno 4294977113 00000004:00080000:1.0:1783520935.711809:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435151a6680, transno 4294977114 00000004:00080000:1.0:1783520935.711813:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435151a5f80, transno 4294977115 00000004:00080000:1.0:1783520935.711817:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435151a4e00, transno 4294977116 00000004:00080000:1.0:1783520935.711821:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435151a5180, transno 4294977117 00000004:00080000:1.0:1783520935.711826:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435151a4700, transno 4294977118 00000004:00080000:1.0:1783520935.711833:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435151a5500, transno 4294977119 00000004:00080000:1.0:1783520935.711838:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943508058000, transno 4294977120 00000004:00080000:1.0:1783520935.711842:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435151a4380, transno 4294977121 00000004:00080000:1.0:1783520935.711846:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cba9500, transno 4294977122 00000004:00080000:1.0:1783520935.711852:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351575c000, transno 4294977123 00000004:00080000:1.0:1783520935.711858:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515307b80, transno 4294977124 00000004:00080000:1.0:1783520935.711862:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515304700, transno 4294977125 00000004:00080000:1.0:1783520935.711866:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515304a80, transno 4294977126 00000100:00100000:3.0:1783520935.711872:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+15:3628:x1870152993054080:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2843us (2915us total) trans 4294975583 rc 0/0 00000004:00080000:1.0:1783520935.711872:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515306d80, transno 4294977127 00000004:00080000:1.0:1783520935.711878:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515306a00, transno 4294977128 00000004:00080000:1.0:1783520935.711883:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515304e00, transno 4294977129 00000100:00100000:3.0:1783520935.711887:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44082 00000004:00080000:1.0:1783520935.711888:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515305f80, transno 4294977130 00000004:00080000:1.0:1783520935.711893:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cbab100, transno 4294977131 00000004:00080000:1.0:1783520935.711897:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cbaa680, transno 4294977132 00000004:00080000:1.0:1783520935.711899:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cbaad80, transno 4294977133 00000004:00080000:1.0:1783520935.711902:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cba9880, transno 4294977134 00000004:00080000:1.0:1783520935.711905:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cba9180, transno 4294977135 00000004:00080000:1.0:1783520935.711907:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350790fb80, transno 4294977136 00000004:00080000:1.0:1783520935.711910:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130f7b80, transno 4294977137 00000004:00080000:1.0:1783520935.711913:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cbaa300, transno 4294977138 00000004:00080000:1.0:1783520935.711915:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130f4380, transno 4294977139 00000004:00080000:1.0:1783520935.711917:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130f7800, transno 4294977140 00000004:00080000:1.0:1783520935.711920:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130f7480, transno 4294977141 00000004:00080000:1.0:1783520935.711924:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130f7100, transno 4294977142 00000004:00080000:1.0:1783520935.711927:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363fe56300, transno 4294977143 00000004:00080000:1.0:1783520935.711929:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130f5500, transno 4294977144 00000004:00080000:1.0:1783520935.711932:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130f4000, transno 4294977145 00000004:00080000:1.0:1783520935.711934:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130f5c00, transno 4294977146 00000004:00080000:1.0:1783520935.711936:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350805aa00, transno 4294977147 00000004:00080000:1.0:1783520935.711939:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435151a7b80, transno 4294977148 00000004:00080000:1.0:1783520935.711942:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943508059c00, transno 4294977149 00000004:00080000:1.0:1783520935.711945:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516358000, transno 4294977150 00000004:00080000:1.0:1783520935.711948:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363d3a6a00, transno 4294977151 00000004:00080000:1.0:1783520935.711950:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94360550d180, transno 4294977152 00000004:00080000:1.0:1783520935.711953:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94360550f100, transno 4294977153 00000004:00080000:1.0:1783520935.711957:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435151a6300, transno 4294977154 00000004:00080000:1.0:1783520935.711961:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130f5f80, transno 4294977155 00000004:00080000:1.0:1783520935.711964:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515301500, transno 4294977156 00000004:00080000:1.0:1783520935.711967:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130f4e00, transno 4294977157 00000004:00080000:1.0:1783520935.711971:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515303b80, transno 4294977158 00000004:00080000:1.0:1783520935.711974:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515300380, transno 4294977159 00000004:00080000:1.0:1783520935.711979:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363840c700, transno 4294977160 00000004:00080000:3.0:1783520935.711983:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517246300/2, rc 0, transno 4294975583 00000004:00080000:1.0:1783520935.711983:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514d29880, transno 4294977161 00000004:00080000:1.0:1783520935.711988:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130f5880, transno 4294977162 00000004:00080000:1.0:1783520935.711992:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130f4700, transno 4294977163 00000100:00100000:3.0:1783520935.711993:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517246300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993054080:0@lo:6:osp-syn-1-0.0 00000004:00080000:1.0:1783520935.711997:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351dc43b80, transno 4294977164 00000004:00080000:1.0:1783520935.712003:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435038edf80, transno 4294977165 00000004:00080000:1.0:1783520935.712006:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943634573b80, transno 4294977166 00000004:00080000:1.0:1783520935.712010:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516310380, transno 4294977167 00000100:00100000:3.0:1783520935.712011:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507acc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993054464:0@lo:6:osp-syn-1-0.0 00000004:00080000:1.0:1783520935.712014:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435151a7480, transno 4294977168 00000004:00080000:1.0:1783520935.712020:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435038ec380, transno 4294977169 00000004:00080000:1.0:1783520935.712024:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cba9c00, transno 4294977170 00000004:00080000:1.0:1783520935.712026:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435118b0e00, transno 4294977171 00000004:00080000:1.0:1783520935.712030:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514d28380, transno 4294977172 00000004:00080000:1.0:1783520935.712034:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cbab480, transno 4294977173 00000004:00080000:1.0:1783520935.712037:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e1c7100, transno 4294977174 00000004:00080000:1.0:1783520935.712040:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e1c5180, transno 4294977175 00000004:00080000:1.0:1783520935.712043:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350790c000, transno 4294977176 00000100:00100000:3.0:1783520935.712048:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:1.0:1783520935.712049:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435151a5880, transno 4294977177 00000004:00080000:1.0:1783520935.712054:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435151a6d80, transno 4294977178 00000004:00080000:1.0:1783520935.712056:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cbab800, transno 4294977179 00000004:00080000:1.0:1783520935.712061:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350790ed80, transno 4294977180 00000004:00080000:1.0:1783520935.712067:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e1c4000, transno 4294977181 00000004:00080000:1.0:1783520935.712070:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e1c5500, transno 4294977182 00000004:00080000:1.0:1783520935.712073:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e1c6680, transno 4294977183 00000004:00080000:1.0:1783520935.712077:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e1c6d80, transno 4294977184 00000100:00100000:3.0:1783520935.712080:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507acfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993054976:0@lo:6:osp-syn-1-0.0 00000004:00080000:1.0:1783520935.712083:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350790f100, transno 4294977185 00000004:00080000:1.0:1783520935.712089:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130f6300, transno 4294977186 00000004:00080000:1.0:1783520935.712093:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350790ea00, transno 4294977187 00000004:00080000:1.0:1783520935.712099:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350790df80, transno 4294977188 00000004:00080000:1.0:1783520935.712103:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350790ce00, transno 4294977189 00000004:00080000:1.0:1783520935.712108:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350790dc00, transno 4294977190 00000004:00080000:1.0:1783520935.712113:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350790e300, transno 4294977191 00000004:00080000:1.0:1783520935.712115:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350790c700, transno 4294977192 00000004:00080000:1.0:1783520935.712120:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350790d180, transno 4294977193 00000100:00100000:3.0:1783520935.712121:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:1.0:1783520935.712125:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350790f480, transno 4294977194 00000004:00080000:1.0:1783520935.712131:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e0f5180, transno 4294977195 00000004:00080000:1.0:1783520935.712135:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435151a4000, transno 4294977196 00000004:00080000:1.0:1783520935.712140:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350790e680, transno 4294977197 00000004:00080000:1.0:1783520935.712146:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350790f800, transno 4294977198 00000004:00080000:1.0:1783520935.712153:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350790d880, transno 4294977199 00000004:00080000:1.0:1783520935.712156:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c56b100, transno 4294977200 00000004:00080000:1.0:1783520935.712160:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c56ce00, transno 4294977201 00000100:00100000:3.0:1783520935.712164:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993054464 00000004:00080000:1.0:1783520935.712165:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c569c00, transno 4294977202 00000100:00100000:3.0:1783520935.712170:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993054464 00000004:00080000:1.0:1783520935.712171:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c56d180, transno 4294977203 00000004:00080000:1.0:1783520935.712177:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c56f800, transno 4294977204 00000004:00080000:1.0:1783520935.712180:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c56ca80, transno 4294977205 00000004:00080000:1.0:1783520935.712185:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c56c000, transno 4294977206 00000004:00080000:1.0:1783520935.712189:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c56c380, transno 4294977207 00000004:00080000:1.0:1783520935.712195:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435038ed180, transno 4294977208 00000004:00080000:1.0:1783520935.712200:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435151a5c00, transno 4294977209 00000100:00100000:3.0:1783520935.712201:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993054976 00000004:00080000:1.0:1783520935.712205:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c56df80, transno 4294977210 00000100:00100000:3.0:1783520935.712206:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993054976 00000004:00080000:1.0:1783520935.712209:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350790c380, transno 4294977211 00000004:00080000:1.0:1783520935.712215:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c56f100, transno 4294977212 00000004:00080000:1.0:1783520935.712221:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c568e00, transno 4294977213 00000004:00080000:1.0:1783520935.712227:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350790d500, transno 4294977214 00000004:00080000:1.0:1783520935.712231:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c569880, transno 4294977215 00000100:00100000:3.0:1783520935.712232:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993054848 00000004:00080000:1.0:1783520935.712235:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c56b800, transno 4294977216 00000100:00100000:3.0:1783520935.712237:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993054848 00000004:00080000:1.0:1783520935.712241:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c569180, transno 4294977217 00000004:00080000:1.0:1783520935.712248:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515776680, transno 4294977218 00000004:00080000:1.0:1783520935.712253:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514dd7b80, transno 4294977219 00000004:00080000:1.0:1783520935.712259:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363d3a7100, transno 4294977220 00000100:00100000:3.0:1783520935.712261:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993054720 00000004:00080000:1.0:1783520935.712265:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350544d880, transno 4294977221 00000100:00100000:3.0:1783520935.712267:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993054720 00000004:00080000:1.0:1783520935.712270:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515304000, transno 4294977222 00000004:00080000:1.0:1783520935.712275:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515307480, transno 4294977223 00000004:00080000:1.0:1783520935.712279:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943508059880, transno 4294977224 00000004:00080000:1.0:1783520935.712282:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515305500, transno 4294977225 00000004:00080000:1.0:1783520935.712285:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363d3a6d80, transno 4294977226 00000004:00080000:1.0:1783520935.712289:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c56ad80, transno 4294977227 00000004:00080000:1.0:1783520935.712293:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c568000, transno 4294977228 00000004:00080000:1.0:1783520935.712299:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c568a80, transno 4294977229 00000004:00080000:1.0:1783520935.712305:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c56bb80, transno 4294977230 00000004:00080000:1.0:1783520935.712309:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c569500, transno 4294977231 00000004:00080000:1.0:1783520935.712313:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130e0380, transno 4294977232 00000004:00080000:1.0:1783520935.712316:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130e2a00, transno 4294977233 00000004:00080000:1.0:1783520935.712319:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130e1f80, transno 4294977234 00000004:00080000:1.0:1783520935.712324:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130e2300, transno 4294977235 00000004:00080000:1.0:1783520935.712330:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e1c4380, transno 4294977236 00000004:00080000:1.0:1783520935.712334:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130e1500, transno 4294977237 00000004:00080000:1.0:1783520935.712337:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130e0700, transno 4294977238 00000004:00080000:1.0:1783520935.712341:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c568380, transno 4294977239 00000004:00080000:1.0:1783520935.712345:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350544d500, transno 4294977240 00000004:00080000:1.0:1783520935.712352:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cba9f80, transno 4294977241 00000100:00100000:3.0:1783520935.712353:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993054592 00000004:00080000:1.0:1783520935.712378:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c56b480, transno 4294977242 00000100:00100000:3.0:1783520935.712379:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993054592 00000004:00080000:1.0:1783520935.712385:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516312680, transno 4294977243 00000004:00080000:1.0:1783520935.712392:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c568700, transno 4294977244 00000004:00080000:1.0:1783520935.712396:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350790ca80, transno 4294977245 00000004:00080000:1.0:1783520935.712400:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503324380, transno 4294977246 00000100:00100000:3.0:1783520935.712402:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993055104 00000004:00080000:1.0:1783520935.712404:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e1c4700, transno 4294977247 00000100:00100000:3.0:1783520935.712406:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993055104 00000004:00080000:1.0:1783520935.712409:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515307800, transno 4294977248 00000004:00080000:1.0:1783520935.712415:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130e2d80, transno 4294977249 00000004:00080000:1.0:1783520935.712420:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515305880, transno 4294977250 00000004:00080000:1.0:1783520935.712426:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e46d500, transno 4294977251 00000004:00080000:1.0:1783520935.712431:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94363840d880, transno 4294977252 00000100:00100000:3.0:1783520935.712436:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44086 00000004:00080000:1.0:1783520935.712437:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435151a7800, transno 4294977253 00000004:00080000:1.0:1783520935.712444:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515305c00, transno 4294977254 00000004:00080000:1.0:1783520935.712448:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e46ea00, transno 4294977255 00000100:00100000:3.0:1783520935.712452:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+20:3628:x1870152993054976:12345-0@lo:6:osp-syn-1-0.0 00000004:00080000:1.0:1783520935.712454:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e46ed80, transno 4294977256 00000004:00080000:1.0:1783520935.712460:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e471880, transno 4294977257 00000004:00080000:1.0:1783520935.712465:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e46df80, transno 4294977258 00000004:00080000:1.0:1783520935.712468:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e46f100, transno 4294977259 00002000:00080000:3.0:1783520935.712471:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1326 count 1 00000004:00080000:1.0:1783520935.712472:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e46c700, transno 4294977260 00000004:00080000:1.0:1783520935.712479:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e472680, transno 4294977261 00000004:00080000:1.0:1783520935.712484:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e46ca80, transno 4294977262 00010000:00010000:3.0:1783520935.712488:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0250c9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x52e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.712490:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e46e680, transno 4294977263 00000004:00080000:1.0:1783520935.712496:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e1c7800, transno 4294977264 00000004:00080000:1.0:1783520935.712515:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515306300, transno 4294977265 00000004:00080000:1.0:1783520935.712522:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e470700, transno 4294977266 00000004:00080000:1.0:1783520935.712529:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c550e00, transno 4294977267 00010000:00010000:3.0:1783520935.712530:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000058fe8463/0xd5401cbabc0229a1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x52e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3091 expref: 979 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.712534:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e46fb80, transno 4294977268 00000004:00080000:1.0:1783520935.712540:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e1c7480, transno 4294977269 00000004:00080000:1.0:1783520935.712546:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e472a00, transno 4294977270 00000004:00080000:1.0:1783520935.712551:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c551c00, transno 4294977271 00010000:00010000:3.0:1783520935.712556:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x52e:0x0].0x0 since=36s v=(0 0) 00000004:00080000:1.0:1783520935.712558:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e471c00, transno 4294977272 00010000:00010000:3.0:1783520935.712562:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x52e:0x0].0x0 since=0s v=(1 0) 00000004:00080000:1.0:1783520935.712563:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e470000, transno 4294977273 00000004:00080000:1.0:1783520935.712570:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c552300, transno 4294977274 00010000:00010000:3.0:1783520935.712572:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0250c9 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x52e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.712575:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e473b80, transno 4294977275 00000004:00080000:1.0:1783520935.712582:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e472300, transno 4294977276 00000004:00080000:1.0:1783520935.712586:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c550380, transno 4294977277 00000004:00080000:1.0:1783520935.712591:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c553800, transno 4294977278 00000004:00080000:1.0:1783520935.712597:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c553b80, transno 4294977279 00000004:00080000:1.0:1783520935.712602:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c551880, transno 4294977280 00000004:00080000:1.0:1783520935.712607:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c551500, transno 4294977281 00000004:00080000:1.0:1783520935.712614:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c556d80, transno 4294977282 00010000:00010000:3.0:1783520935.712615:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000058fe8463/0xd5401cbabc0229a1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x52e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3091 expref: 979 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.712621:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c554700, transno 4294977283 00000004:00080000:1.0:1783520935.712627:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c554a80, transno 4294977284 00000004:00080000:1.0:1783520935.712633:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c554380, transno 4294977285 00010000:00010000:3.0:1783520935.712639:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000058fe8463/0xd5401cbabc0229a1 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x52e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3091 expref: 979 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.712639:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c56c700, transno 4294977286 00000004:00080000:1.0:1783520935.712646:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c556a00, transno 4294977287 00000004:00080000:1.0:1783520935.712653:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c56d500, transno 4294977288 00000100:00100000:3.0:1783520935.712658:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530f800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993055360:192.168.204.33@tcp:104: 00000004:00080000:1.0:1783520935.712660:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c557100, transno 4294977289 00000004:00080000:1.0:1783520935.712666:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c555c00, transno 4294977290 00000004:00080000:1.0:1783520935.712672:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e1c4e00, transno 4294977291 00000004:00080000:1.0:1783520935.712676:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c56d880, transno 4294977292 00000004:00080000:1.0:1783520935.712680:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c557480, transno 4294977293 00000004:00080000:1.0:1783520935.712684:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c556680, transno 4294977294 00000004:00080000:1.0:1783520935.712687:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e47ad80, transno 4294977295 00000004:00080000:1.0:1783520935.712692:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e1c7b80, transno 4294977296 00000100:00100000:3.0:1783520935.712694:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872e40 going to sleep for 11 seconds 00000004:00080000:1.0:1783520935.712696:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c555f80, transno 4294977297 00000004:00080000:1.0:1783520935.712702:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e47aa00, transno 4294977298 00000004:00080000:1.0:1783520935.712706:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c556300, transno 4294977299 00000004:00080000:1.0:1783520935.712709:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e478a80, transno 4294977300 00000004:00080000:1.0:1783520935.712713:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e472d80, transno 4294977301 00000004:00080000:1.0:1783520935.712718:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e478380, transno 4294977302 00000100:00100000:3.0:1783520935.712721:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44087 00000004:00080000:1.0:1783520935.712725:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e479180, transno 4294977303 00000004:00080000:1.0:1783520935.712730:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e471180, transno 4294977304 00000100:00100000:3.0:1783520935.712733:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507acce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+20:3627:x1870152993054848:12345-0@lo:6:osp-syn-1-0.0 00000004:00080000:1.0:1783520935.712735:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e478e00, transno 4294977305 00000004:00080000:1.0:1783520935.712742:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e47b100, transno 4294977306 00000004:00080000:1.0:1783520935.712747:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e478000, transno 4294977307 00000004:00080000:1.0:1783520935.712751:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e470380, transno 4294977308 00002000:00080000:3.0:1783520935.712753:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1325 count 1 00000004:00080000:1.0:1783520935.712755:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e479f80, transno 4294977309 00000004:00080000:1.0:1783520935.712761:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e471500, transno 4294977310 00000004:00080000:1.0:1783520935.712765:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503849180, transno 4294977311 00010000:00010000:3.0:1783520935.712767:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc0250d0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x52d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.712769:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e470a80, transno 4294977312 00000004:00080000:1.0:1783520935.712775:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e473480, transno 4294977313 00000004:00080000:1.0:1783520935.712780:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435121b2680, transno 4294977314 00000004:00080000:1.0:1783520935.712784:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435121b0700, transno 4294977315 00010000:00010000:3.0:1783520935.712785:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000fc8540e3/0xd5401cbabc01f257 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x52d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859ef95c expref: 979 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.712789:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435121b0a80, transno 4294977316 00000004:00080000:1.0:1783520935.712795:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435121b2d80, transno 4294977317 00000004:00080000:1.0:1783520935.712801:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435121b3480, transno 4294977318 00000004:00080000:1.0:1783520935.712807:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c553100, transno 4294977319 00000004:00080000:1.0:1783520935.712813:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c551f80, transno 4294977320 00010000:00010000:3.0:1783520935.712814:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x52d:0x0].0x0 since=37s v=(0 0) 00000004:00080000:1.0:1783520935.712817:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435121b1c00, transno 4294977321 00010000:00010000:3.0:1783520935.712819:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x52d:0x0].0x0 since=0s v=(0 0) 00000004:00080000:1.0:1783520935.712824:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435121b1180, transno 4294977322 00010000:00010000:3.0:1783520935.712825:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc0250d0 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x52d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.712830:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c557b80, transno 4294977323 00000004:00080000:1.0:1783520935.712835:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c552d80, transno 4294977324 00000004:00080000:1.0:1783520935.712841:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c555880, transno 4294977325 00000004:00080000:1.0:1783520935.712845:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c557800, transno 4294977326 00000004:00080000:1.0:1783520935.712850:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c554000, transno 4294977327 00010000:00010000:3.0:1783520935.712853:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000fc8540e3/0xd5401cbabc01f257 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x52d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef95c expref: 979 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.712855:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350792f480, transno 4294977328 00000004:00080000:1.0:1783520935.712860:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350792d180, transno 4294977329 00000004:00080000:1.0:1783520935.712865:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350792d880, transno 4294977330 00000004:00080000:1.0:1783520935.712869:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e471f80, transno 4294977331 00000004:00080000:1.0:1783520935.712872:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350792c380, transno 4294977332 00000004:00080000:1.0:1783520935.712874:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350793b800, transno 4294977333 00000004:00080000:1.0:1783520935.712878:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350792c700, transno 4294977334 00000004:00080000:1.0:1783520935.712882:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350792f100, transno 4294977335 00000004:00080000:1.0:1783520935.712887:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350792ed80, transno 4294977336 00000004:00080000:1.0:1783520935.712891:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350792c000, transno 4294977337 00000004:00080000:1.0:1783520935.712896:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350792ea00, transno 4294977338 00000004:00080000:1.0:1783520935.712899:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350792dc00, transno 4294977339 00000004:00080000:1.0:1783520935.712903:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c555180, transno 4294977340 00000004:00080000:1.0:1783520935.712907:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350793ed80, transno 4294977341 00000004:00080000:1.0:1783520935.712913:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350793ca80, transno 4294977342 00000004:00080000:1.0:1783520935.712918:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350793dc00, transno 4294977343 00000004:00080000:1.0:1783520935.712923:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350793ce00, transno 4294977344 00000004:00080000:1.0:1783520935.712930:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350793f480, transno 4294977345 00000004:00080000:1.0:1783520935.712935:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350793df80, transno 4294977346 00000004:00080000:1.0:1783520935.712941:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350793d180, transno 4294977347 00000004:00080000:1.0:1783520935.712947:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350793e300, transno 4294977348 00010000:00010000:3.0:1783520935.712948:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000fc8540e3/0xd5401cbabc01f257 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x52d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef95c expref: 980 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.712953:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350793e680, transno 4294977349 00000004:00080000:1.0:1783520935.712959:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350793f800, transno 4294977350 00000004:00080000:1.0:1783520935.712965:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350793c380, transno 4294977351 00000004:00080000:1.0:1783520935.712969:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350793ea00, transno 4294977352 00000004:00080000:1.0:1783520935.712976:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e479c00, transno 4294977353 00000004:00080000:1.0:1783520935.712980:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350793d500, transno 4294977354 00000100:00100000:3.0:1783520935.712981:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530c700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993055488:192.168.204.33@tcp:104: 00000004:00080000:1.0:1783520935.712983:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130e3100, transno 4294977355 00000004:00080000:1.0:1783520935.712990:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e479880, transno 4294977356 00000004:00080000:1.0:1783520935.712995:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e479500, transno 4294977357 00000004:00080000:1.0:1783520935.713001:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130e0000, transno 4294977358 00000004:00080000:1.0:1783520935.713005:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130e3b80, transno 4294977359 00000004:00080000:1.0:1783520935.713010:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cbaaa00, transno 4294977360 00000004:00080000:1.0:1783520935.713015:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cbabb80, transno 4294977361 00000004:00080000:1.0:1783520935.713020:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515307100, transno 4294977362 00000100:00100000:3.0:1783520935.713022:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872f00 going to sleep for 11 seconds 00000004:00080000:1.0:1783520935.713026:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351cba8700, transno 4294977363 00000004:00080000:1.0:1783520935.713032:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515305180, transno 4294977364 00000004:00080000:1.0:1783520935.713038:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e46d180, transno 4294977365 00000004:00080000:1.0:1783520935.713044:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516359f80, transno 4294977366 00000004:00080000:1.0:1783520935.713050:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943509a83480, transno 4294977367 00000004:00080000:1.0:1783520935.713053:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e46f480, transno 4294977368 00000004:00080000:1.0:1783520935.713057:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9436342bad80, transno 4294977369 00000004:00080000:1.0:1783520935.713061:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e46d880, transno 4294977370 00000004:00080000:1.0:1783520935.713067:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515306680, transno 4294977371 00000004:00080000:1.0:1783520935.713072:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e46c380, transno 4294977372 00000004:00080000:1.0:1783520935.713076:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c56a300, transno 4294977373 00000004:00080000:1.0:1783520935.713081:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e47a300, transno 4294977374 00000004:00080000:1.0:1783520935.713086:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350793aa00, transno 4294977375 00000004:00080000:1.0:1783520935.713092:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c56aa00, transno 4294977376 00000004:00080000:1.0:1783520935.713099:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514d2aa00, transno 4294977377 00000004:00080000:1.0:1783520935.713103:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515f09500, transno 4294977378 00000004:00080000:1.0:1783520935.713108:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130f5180, transno 4294977379 00000004:00080000:1.0:1783520935.713112:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515f08a80, transno 4294977380 00000100:00100000:3.0:1783520935.713115:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44088 00000004:00080000:1.0:1783520935.713118:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e46f800, transno 4294977381 00000004:00080000:1.0:1783520935.713122:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515f0bb80, transno 4294977382 00000100:00100000:3.0:1783520935.713126:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435166c8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+20:3626:x1870152993054720:12345-0@lo:6:osp-syn-1-0.0 00000004:00080000:1.0:1783520935.713128:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515f09c00, transno 4294977383 00000004:00080000:1.0:1783520935.713133:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515f08e00, transno 4294977384 00000004:00080000:1.0:1783520935.713139:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515301c00, transno 4294977385 00000004:00080000:1.0:1783520935.713143:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515f0ad80, transno 4294977386 00000004:00080000:1.0:1783520935.713146:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515302d80, transno 4294977387 00000004:00080000:1.0:1783520935.713153:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515303100, transno 4294977388 00000004:00080000:1.0:1783520935.713157:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515300a80, transno 4294977389 00000004:00080000:1.0:1783520935.713162:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515302a00, transno 4294977390 00000004:00080000:1.0:1783520935.713167:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515301880, transno 4294977391 00000004:00080000:1.0:1783520935.713171:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515300000, transno 4294977392 00000004:00080000:1.0:1783520935.713175:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515301f80, transno 4294977393 00000004:00080000:1.0:1783520935.713181:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515303800, transno 4294977394 00000004:00080000:1.0:1783520935.713186:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515300e00, transno 4294977395 00002000:00080000:3.0:1783520935.713191:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1324 count 1 00000004:00080000:1.0:1783520935.713191:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515302300, transno 4294977396 00000004:00080000:1.0:1783520935.713198:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130e2680, transno 4294977397 00000004:00080000:1.0:1783520935.713203:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130e1880, transno 4294977398 00010000:00010000:3.0:1783520935.713204:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0250de lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x52c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.713208:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350793fb80, transno 4294977399 00000004:00080000:1.0:1783520935.713214:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350ade2300, transno 4294977400 00000004:00080000:1.0:1783520935.713218:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e46ce00, transno 4294977401 00000004:00080000:1.0:1783520935.713226:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e452a00, transno 4294977402 00010000:00010000:3.0:1783520935.713229:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000000415af11/0xd5401cbabc01f249 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x52c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859ef94e expref: 978 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.713231:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e46c000, transno 4294977403 00000004:00080000:1.0:1783520935.713237:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e453100, transno 4294977404 00000004:00080000:1.0:1783520935.713240:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350793c000, transno 4294977405 00000004:00080000:1.0:1783520935.713244:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e46dc00, transno 4294977406 00000004:00080000:1.0:1783520935.713249:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c569f80, transno 4294977407 00000004:00080000:1.0:1783520935.713253:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e450e00, transno 4294977408 00000004:00080000:1.0:1783520935.713258:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e451880, transno 4294977409 00000004:00080000:1.0:1783520935.713263:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e453800, transno 4294977410 00000004:00080000:1.0:1783520935.713269:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94360550c380, transno 4294977411 00000004:00080000:1.0:1783520935.713274:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e453b80, transno 4294977412 00000004:00080000:1.0:1783520935.713279:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e452300, transno 4294977413 00000004:00080000:1.0:1783520935.713286:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e451180, transno 4294977414 00010000:00010000:3.0:1783520935.713287:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x52c:0x0].0x0 since=37s v=(0 0) 00000004:00080000:1.0:1783520935.713292:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507ae5c00, transno 4294977415 00010000:00010000:3.0:1783520935.713295:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x52c:0x0].0x0 since=0s v=(0 0) 00000004:00080000:1.0:1783520935.713299:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c554e00, transno 4294977416 00010000:00010000:3.0:1783520935.713302:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0250de lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x52c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.713304:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c555500, transno 4294977417 00000004:00080000:1.0:1783520935.713310:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e450380, transno 4294977418 00000004:00080000:1.0:1783520935.713316:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507ae5880, transno 4294977419 00000004:00080000:1.0:1783520935.713320:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507ae7800, transno 4294977420 00000004:00080000:1.0:1783520935.713324:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515f0aa00, transno 4294977421 00000004:00080000:1.0:1783520935.713327:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507ae7b80, transno 4294977422 00010000:00010000:3.0:1783520935.713331:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000000415af11/0xd5401cbabc01f249 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x52c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef94e expref: 978 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.713333:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507ae5180, transno 4294977423 00000004:00080000:1.0:1783520935.713341:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507ae6300, transno 4294977424 00000004:00080000:1.0:1783520935.713349:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d91f80, transno 4294977425 00010000:00010000:3.0:1783520935.713350:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000000415af11/0xd5401cbabc01f249 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x52c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef94e expref: 978 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.713356:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515f08380, transno 4294977426 00000100:00100000:3.0:1783520935.713383:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530c380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993055744:192.168.204.33@tcp:104: 00000004:00080000:1.0:1783520935.713395:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507ae7480, transno 4294977427 00000004:00080000:1.0:1783520935.713401:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d93800, transno 4294977428 00000004:00080000:1.0:1783520935.713405:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d90000, transno 4294977429 00000004:00080000:1.0:1783520935.713408:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d92300, transno 4294977430 00000004:00080000:1.0:1783520935.713413:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d90700, transno 4294977431 00000004:00080000:1.0:1783520935.713419:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d90a80, transno 4294977432 00000100:00100000:3.0:1783520935.713423:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872480 going to sleep for 11 seconds 00000004:00080000:1.0:1783520935.713425:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d91880, transno 4294977433 00000004:00080000:1.0:1783520935.713432:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d90e00, transno 4294977434 00000004:00080000:1.0:1783520935.713437:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d91c00, transno 4294977435 00000004:00080000:1.0:1783520935.713443:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515f09f80, transno 4294977436 00000004:00080000:1.0:1783520935.713449:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515f0a300, transno 4294977437 00000004:00080000:1.0:1783520935.713453:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515f08700, transno 4294977438 00000004:00080000:1.0:1783520935.713457:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515302680, transno 4294977439 00000004:00080000:1.0:1783520935.713461:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515f09180, transno 4294977440 00000004:00080000:1.0:1783520935.713466:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515f0b480, transno 4294977441 00000004:00080000:1.0:1783520935.713470:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515f0b100, transno 4294977442 00000004:00080000:1.0:1783520935.713474:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515f0b800, transno 4294977443 00000004:00080000:1.0:1783520935.713477:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515301180, transno 4294977444 00000004:00080000:1.0:1783520935.713479:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515300700, transno 4294977445 00000004:00080000:1.0:1783520935.713483:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130f4a80, transno 4294977446 00000004:00080000:1.0:1783520935.713488:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350793c700, transno 4294977447 00000004:00080000:1.0:1783520935.713494:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350792ce00, transno 4294977448 00000004:00080000:1.0:1783520935.713515:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e46e300, transno 4294977449 00000004:00080000:1.0:1783520935.713520:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350793f100, transno 4294977450 00000004:00080000:1.0:1783520935.713526:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350792fb80, transno 4294977451 00000004:00080000:1.0:1783520935.713532:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350792df80, transno 4294977452 00000004:00080000:1.0:1783520935.713538:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350792f800, transno 4294977453 00000004:00080000:1.0:1783520935.713543:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350792d500, transno 4294977454 00000004:00080000:1.0:1783520935.713549:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d9f100, transno 4294977455 00000004:00080000:1.0:1783520935.713554:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c552a00, transno 4294977456 00000004:00080000:1.0:1783520935.713559:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d9df80, transno 4294977457 00000004:00080000:1.0:1783520935.713562:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d9a300, transno 4294977458 00000004:00080000:1.0:1783520935.713566:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d9fb80, transno 4294977459 00000004:00080000:1.0:1783520935.713571:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c552680, transno 4294977460 00000004:00080000:1.0:1783520935.713579:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d9bb80, transno 4294977461 00000004:00080000:1.0:1783520935.713583:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d9c000, transno 4294977462 00000004:00080000:1.0:1783520935.713589:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e473100, transno 4294977463 00000004:00080000:1.0:1783520935.713592:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e1c6a00, transno 4294977464 00000004:00080000:1.0:1783520935.713596:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d98a80, transno 4294977465 00000004:00080000:1.0:1783520935.713600:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d9d880, transno 4294977466 00000004:00080000:1.0:1783520935.713603:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d98e00, transno 4294977467 00000004:00080000:1.0:1783520935.713607:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d9aa00, transno 4294977468 00000004:00080000:1.0:1783520935.713611:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c553480, transno 4294977469 00000004:00080000:1.0:1783520935.713615:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d9c380, transno 4294977470 00000004:00080000:1.0:1783520935.713618:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d9d500, transno 4294977471 00000004:00080000:1.0:1783520935.713621:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c550a80, transno 4294977472 00000004:00080000:1.0:1783520935.713625:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c56ed80, transno 4294977473 00000004:00080000:1.0:1783520935.713629:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c56dc00, transno 4294977474 00000004:00080000:1.0:1783520935.713633:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943503849f80, transno 4294977475 00000004:00080000:1.0:1783520935.713636:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350792e300, transno 4294977476 00000004:00080000:1.0:1783520935.713641:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515304380, transno 4294977477 00000004:00080000:1.0:1783520935.713645:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e453480, transno 4294977478 00000004:00080000:1.0:1783520935.713652:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c56e680, transno 4294977479 00000004:00080000:1.0:1783520935.713658:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e451f80, transno 4294977480 00000004:00080000:1.0:1783520935.713664:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943516682680, transno 4294977481 00000004:00080000:1.0:1783520935.713670:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e452d80, transno 4294977482 00000004:00080000:1.0:1783520935.713675:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e451500, transno 4294977483 00000004:00080000:1.0:1783520935.713681:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e450000, transno 4294977484 00000100:00100000:3.0:1783520935.713687:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e0f4a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993055232:192.168.204.33@tcp:104: 00000004:00080000:1.0:1783520935.713687:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e473800, transno 4294977485 00000004:00080000:1.0:1783520935.713695:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d99880, transno 4294977486 00000004:00080000:1.0:1783520935.713701:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d99180, transno 4294977487 00000004:00080000:1.0:1783520935.713706:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c56a680, transno 4294977488 00010000:00010000:3.0:1783520935.713708:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000004b9eec72/0xd5401cbabc0250c2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x52a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.713709:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350805a300, transno 4294977489 00000004:00080000:1.0:1783520935.713715:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d93480, transno 4294977490 00000004:00080000:1.0:1783520935.713721:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d92a00, transno 4294977491 00000004:00080000:1.0:1783520935.713725:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d91180, transno 4294977492 00010000:00010000:3.0:1783520935.713728:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000004:00080000:1.0:1783520935.713729:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e47b800, transno 4294977493 00010000:00010000:3.0:1783520935.713734:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000003f15af42/0xd5401cbabc0250b4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x528:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.713736:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d90380, transno 4294977494 00000004:00080000:1.0:1783520935.713742:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e47a680, transno 4294977495 00010000:00010000:3.0:1783520935.713743:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf7000 already unused, calling callback (ffffffffc0bf9e90) 00000004:00080000:1.0:1783520935.713746:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d92680, transno 4294977496 00010000:00010000:3.0:1783520935.713748:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000003f15af42/0xd5401cbabc0250b4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x528:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.713750:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e47bb80, transno 4294977497 00000004:00080000:1.0:1783520935.713754:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d9b480, transno 4294977498 00010000:00010000:3.0:1783520935.713758:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000003f15af42/0xd5401cbabc0250b4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x528:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.713758:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d98380, transno 4294977499 00000004:00080000:1.0:1783520935.713765:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d9a680, transno 4294977500 00000004:00080000:1.0:1783520935.713772:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435121b0e00, transno 4294977501 00010000:00010000:3.0:1783520935.713777:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000003f15af42/0xd5401cbabc0250b4 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x528:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.713778:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435121b1880, transno 4294977502 00000004:00080000:1.0:1783520935.713784:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d99c00, transno 4294977503 00000004:00080000:1.0:1783520935.713791:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d9ad80, transno 4294977504 00010000:00010000:3.0:1783520935.713793:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003f15af42/0xd5401cbabc0250b4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x528:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.713798:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d9b800, transno 4294977505 00000004:00080000:1.0:1783520935.713805:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507939180, transno 4294977506 00000004:00080000:1.0:1783520935.713812:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507939f80, transno 4294977507 00000004:00080000:1.0:1783520935.713815:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507938380, transno 4294977508 00000004:00080000:1.0:1783520935.713818:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507938e00, transno 4294977509 00000004:00080000:1.0:1783520935.713821:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507938700, transno 4294977510 00000004:00080000:1.0:1783520935.713824:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350793b480, transno 4294977511 00000004:00080000:1.0:1783520935.713827:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350793ad80, transno 4294977512 00000004:00080000:1.0:1783520935.713830:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943515303480, transno 4294977513 00000004:00080000:1.0:1783520935.713834:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e451c00, transno 4294977514 00000004:00080000:1.0:1783520935.713838:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350793d880, transno 4294977515 00000004:00080000:1.0:1783520935.713843:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c56ea00, transno 4294977516 00000004:00080000:1.0:1783520935.713847:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435121b3b80, transno 4294977517 00000004:00080000:1.0:1783520935.713851:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d9f800, transno 4294977518 00000004:00080000:1.0:1783520935.713854:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d9ed80, transno 4294977519 00000004:00080000:1.0:1783520935.713857:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d9ca80, transno 4294977520 00000004:00080000:1.0:1783520935.713861:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350792ca80, transno 4294977521 00000004:00080000:1.0:1783520935.713865:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c550000, transno 4294977522 00000004:00080000:1.0:1783520935.713869:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d9e680, transno 4294977523 00000004:00080000:1.0:1783520935.713872:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435121b2300, transno 4294977524 00000004:00080000:1.0:1783520935.713876:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d9dc00, transno 4294977525 00000004:00080000:1.0:1783520935.713881:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d9f480, transno 4294977526 00000004:00080000:1.0:1783520935.713886:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350c551180, transno 4294977527 00000004:00080000:1.0:1783520935.713889:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e1c4a80, transno 4294977528 00000004:00080000:1.0:1783520935.713891:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e1c6300, transno 4294977529 00000004:00080000:1.0:1783520935.713894:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e1c5c00, transno 4294977530 00000004:00080000:1.0:1783520935.713896:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d9ce00, transno 4294977531 00000004:00080000:1.0:1783520935.713897:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94351e1c5880, transno 4294977532 00000004:00080000:1.0:1783520935.713899:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507ad2d80, transno 4294977533 00000004:00080000:1.0:1783520935.713901:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507ad2a00, transno 4294977534 00000004:00080000:1.0:1783520935.713902:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507ad3480, transno 4294977535 00000004:00080000:1.0:1783520935.713904:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507ad1180, transno 4294977536 00000004:00080000:1.0:1783520935.713906:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507ad1880, transno 4294977537 00000004:00080000:1.0:1783520935.713909:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507ad0e00, transno 4294977538 00000004:00080000:1.0:1783520935.713913:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507ad0a80, transno 4294977539 00000004:00080000:1.0:1783520935.713915:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507ad0380, transno 4294977540 00000004:00080000:1.0:1783520935.713918:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507ad3800, transno 4294977541 00000004:00080000:1.0:1783520935.713920:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507ad1c00, transno 4294977542 00000004:00080000:1.0:1783520935.713923:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507acd500, transno 4294977543 00000004:00080000:1.0:1783520935.713927:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507acea00, transno 4294977544 00000004:00080000:1.0:1783520935.713932:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff9435130f6680, transno 4294977545 00000004:00080000:1.0:1783520935.713936:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943506d9c700, transno 4294977546 00000004:00080000:1.0:1783520935.713940:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943514e98a80, transno 4294977547 00000004:00080000:1.0:1783520935.713944:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff94350793a300, transno 4294977548 00000004:00080000:1.0:1783520935.713951:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507acc000, transno 4294977549 00000004:00080000:1.0:1783520935.713957:0:3625:0:(osp_sync.c:570:osp_sync_request_commit_cb()) commit req ffff943507939500, transno 4294977550 00000100:00100000:1.0:1783520935.713966:0:3625:0:(client.c:3156:ptlrpc_free_committed()) @@@ stopping search req@ffff94351e004a80 x1870152993051136/t4294977552(4294977552) o6->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 544/432 e 0 to 0 dl 1783520951 ref 2 fl Rpc:RQU/204/0 rc 0/0 job:'osp-syn-0-0.0' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1783520935.713973:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.713977:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.713979:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520935.713987:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e004a80/2, rc 0, transno 4294977552 00000100:00100000:1.0:1783520935.713995:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e004a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993051136:0@lo:6:osp-syn-0-0.0 00000100:00100000:3.0:1783520935.714003:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530c700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993055488:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.714023:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc0250d0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x52d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.714040:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ace680/2, rc 0, transno 4294975581 00000100:00100000:1.0:1783520935.714053:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ace680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993053696:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.714063:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530f800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993055360:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.714085:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0250c9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x52e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.714176:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.714214:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514f9b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993056256:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.714252:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.714285:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002363648 00000100:00100000:3.0:1783520935.714290:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002363648 00000100:00100000:1.0:1783520935.714302:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993056256 00000100:00100000:1.0:1783520935.714307:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993056256 00000100:00100000:1.0:1783520935.714345:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44092 00000100:00100000:1.0:1783520935.714356:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+22:3627:x1870152993056256:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.714389:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000058fe8463/0xd5401cbabc0229a1 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x52e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3091 expref: 980 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.714400:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1329 count 1 00010000:00010000:1.0:1783520935.714416:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0250f3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x531:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.714424:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35678 00010000:00010000:1.0:1783520935.714435:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000089a7297d/0xd5401cbabc01f28f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x531:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859ef994 expref: 980 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.714439:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530f800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+980:2388:x1870153002363648:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.714452:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x531:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.714457:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x531:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.714462:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0250f3 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x531:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.714486:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000089a7297d/0xd5401cbabc01f28f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x531:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef994 expref: 981 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.714511:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000089a7297d/0xd5401cbabc01f28f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x531:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef994 expref: 981 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.714532:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.714533:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad2680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993056384:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.714543:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000058fe8463/0xd5401cbabc0229a1 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x52e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3091 expref: 981 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.714557:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000058fe8463/0xd5401cbabc0229a1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x52e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3091 expref: 981 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.714575:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000058fe8463/0xd5401cbabc0229a1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x52e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3091 expref: 981 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.714580:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6300 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.714597:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x52e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520935.714603:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x52e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520935.714610:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0250c9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x52e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.714626:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0250c9 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x52e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.714650:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.714664:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0250c9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x52e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.714674:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0250c9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x52e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.714676:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530f800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+979:2388:x1870153002363648:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 241us (500us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.714691:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0250c9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x52e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.714695:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35678 00010000:00010000:1.0:1783520935.714705:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0250c9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x52e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.714727:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.714741:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.714748:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0250c9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x52e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.714825:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.714832:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0250c9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x52e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.714834:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+23:3628:x1870152993054976:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2386us (2713us total) trans 4294975586 rc 0/0 00010000:00010000:3.0:1783520935.714856:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0250c9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x52e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.714860:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44086 00000004:00080000:1.0:1783520935.714907:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507acfb80/2, rc 0, transno 4294975586 00000100:00100000:1.0:1783520935.714922:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507acfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993054976:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.714949:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0250c9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x52e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.714964:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0250c9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x52e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.714972:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35680 00000100:00100000:1.0:1783520935.714988:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514f98e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+980:2388:x1870153002363904:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.714994:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.714998:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.715001:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.715011:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000fc8540e3/0xd5401cbabc01f257 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x52d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef95c expref: 980 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.715029:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000fc8540e3/0xd5401cbabc01f257 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x52d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef95c expref: 980 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.715049:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000fc8540e3/0xd5401cbabc01f257 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x52d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef95c expref: 980 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.715064:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.715066:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x52d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.715073:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x52d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.715081:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc0250d0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x52d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.715093:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530c380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993055744:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.715104:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc0250d0 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x52d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.715117:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0250de lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x52c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.715130:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.715143:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000004b9eec72/0xd5401cbabc0250c2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x52a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.715155:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000004b9eec72/0xd5401cbabc0250c2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x52a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.715162:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514f98e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+978:2388:x1870153002363904:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 175us (354us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.715183:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35680 00010000:00010000:3.0:1783520935.715185:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000004b9eec72/0xd5401cbabc0250c2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x52a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.715195:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000004b9eec72/0xd5401cbabc0250c2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x52a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.715212:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.715213:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc0250d0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x52d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.715224:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc0250d0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x52d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.715233:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc0250d0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x52d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.715241:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc0250d0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x52d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.715258:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520935.715377:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+24:3628:x1870152993054464:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2928us (3310us total) trans 4294975587 rc 0/0 00000100:00100000:1.0:1783520935.715387:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507acce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+24:3627:x1870152993054848:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2656us (3243us total) trans 4294975588 rc 0/0 00000100:00100000:3.0:1783520935.715389:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44085 00000100:00100000:1.0:1783520935.715403:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44087 00000100:00100000:3.0:1783520935.715448:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000004:00080000:1.0:1783520935.715461:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507acc380/2, rc 0, transno 4294975587 00000100:00100000:1.0:1783520935.715470:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507acc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993054464:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.715483:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514f9bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993056512:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.715526:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:1.0:1783520935.715571:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507aced80/2, rc 0, transno 4294975588 00000100:00100000:1.0:1783520935.715584:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507aced80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993054848:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.715610:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002364032 00000100:00100000:1.0:1783520935.715612:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993056512 00000100:00100000:3.0:1783520935.715616:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002364032 00000100:00100000:1.0:1783520935.715618:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993056512 00010000:00010000:3.0:1783520935.715638:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000dbdfad6c/0xd5401cbabc01f23b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x52b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef940 expref: 977 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.715658:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44093 00000100:00100000:3.0:1783520935.715672:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35681 00000100:00100000:1.0:1783520935.715673:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+23:3628:x1870152993056512:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.715688:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530c380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+977:2387:x1870153002364032:12345-192.168.204.33@tcp:103:ldlm_bl.0 00002000:00080000:1.0:1783520935.715691:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1330 count 1 00010000:00010000:3.0:1783520935.715700:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.715707:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc0250fa lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x532:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.715708:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000dbdfad6c/0xd5401cbabc01f23b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x52b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef940 expref: 977 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.715728:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d355441b/0xd5401cbabc01f29d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x532:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9a2 expref: 977 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.715736:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x532:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.715741:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x532:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.715748:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc0250fa lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x532:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.715776:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000dbdfad6c/0xd5401cbabc01f23b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x52b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef940 expref: 977 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.715780:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d355441b/0xd5401cbabc01f29d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x532:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9a2 expref: 977 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.715809:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d355441b/0xd5401cbabc01f29d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x532:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9a2 expref: 977 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.715818:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000dbdfad6c/0xd5401cbabc01f23b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x52b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef940 expref: 977 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.715834:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad3b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993056896:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.715837:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x52b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.715846:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x52b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.715851:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e23a9c85/0xd5401cbabc0250d7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x52b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.715860:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e23a9c85/0xd5401cbabc0250d7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x52b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.715880:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.715898:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6f00 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.715904:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530c380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+976:2387:x1870153002364032:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 220us (460us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.715918:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35681 00000100:00100000:1.0:1783520935.715924:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44095 00000100:00100000:1.0:1783520935.715938:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514f9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+25:3626:x1870152993056640:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.715958:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1331 count 1 00010000:00010000:1.0:1783520935.715973:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025108 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x533:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.715987:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000cbc4810a/0xd5401cbabc01f2ab lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x533:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9b0 expref: 975 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.716006:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.716006:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x533:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.716012:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x533:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.716015:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025108 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x533:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.716039:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000cbc4810a/0xd5401cbabc01f2ab lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x533:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9b0 expref: 975 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.716049:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000cbc4810a/0xd5401cbabc01f2ab lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x533:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9b0 expref: 975 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.716062:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993057152:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.716085:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.716101:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6240 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.716199:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.716259:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.716281:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002364416 00000100:00100000:1.0:1783520935.716285:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002364416 00010000:00010000:1.0:1783520935.716302:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e57033ee/0xd5401cbabc01f281 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x530:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef986 expref: 977 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.716323:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002364544 00000100:00100000:1.0:1783520935.716328:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35683 00000100:00100000:3.0:1783520935.716330:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002364544 00000100:00100000:1.0:1783520935.716346:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530fb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+978:2387:x1870153002364160:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.716353:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000089a7297d/0xd5401cbabc01f28f lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x531:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef994 expref: 978 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.716380:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.716388:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000000415af11/0xd5401cbabc01f249 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x52c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef94e expref: 977 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.716404:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000000415af11/0xd5401cbabc01f249 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x52c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef94e expref: 977 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.716429:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000415af11/0xd5401cbabc01f249 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x52c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef94e expref: 977 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.716445:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x52c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.716449:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x52c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.716456:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0250de lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x52c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.716468:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0250de lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x52c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.716494:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.716532:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530fb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+975:2387:x1870153002364160:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 191us (448us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.716534:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35685 00000100:00100000:3.0:1783520935.716550:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad2d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+975:2388:x1870153002364544:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.716551:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35683 00010000:00010000:3.0:1783520935.716563:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.716572:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000089a7297d/0xd5401cbabc01f28f lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x531:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef994 expref: 974 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.716586:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000089a7297d/0xd5401cbabc01f28f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x531:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef994 expref: 973 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.716587:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0250de lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x52c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.716612:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x531:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.716622:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x531:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.716626:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0250de lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x52c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.716633:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0250f3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x531:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.716645:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0250de lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x52c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.716661:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0250f3 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x531:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.716662:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0250de lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x52c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.716698:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.716699:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:3.0:1783520935.716724:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad2d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+973:2388:x1870153002364544:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 178us (465us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.716733:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35685 00010000:00010000:3.0:1783520935.716758:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520935.716765:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e23a9c85/0xd5401cbabc0250d7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x52b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.716774:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf7800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.716779:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e23a9c85/0xd5401cbabc0250d7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x52b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.716806:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435166c8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+26:3626:x1870152993054720:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3681us (4586us total) trans 4294975590 rc 0/0 00000100:00100000:1.0:1783520935.716821:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44088 00010000:00010000:3.0:1783520935.716860:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e23a9c85/0xd5401cbabc0250d7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x52b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.716867:0:35276:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000089a7297d/0xd5401cbabc01f28f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x531:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef994 expref: 972 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.716878:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e23a9c85/0xd5401cbabc0250d7 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x52b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.716884:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad2680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993056384:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.716895:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e23a9c85/0xd5401cbabc0250d7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x52b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.716906:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0250f3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x531:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.716916:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0250f3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x531:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.716919:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520935.716926:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0250de lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x52c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.716929:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0250f3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x531:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.716951:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0250f3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x531:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.716962:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0250f3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x531:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.716978:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.716995:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.717002:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0250de lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x52c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.717019:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0250de lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x52c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.717045:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0250de lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x52c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.717064:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0250de lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x52c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.717066:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+26:3627:x1870152993056256:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2711us (2813us total) trans 4294975592 rc 0/0 00000100:00100000:1.0:1783520935.717086:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44092 00010000:00010000:3.0:1783520935.717103:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.717106:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0250f3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x531:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.717114:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.717118:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0250f3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x531:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.717125:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0250f3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x531:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.717125:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514f9b480/2, rc 0, transno 4294975592 00000100:00100000:1.0:1783520935.717138:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514f9b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993056256:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.717183:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0250f3 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x531:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.717184:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.717193:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0250f3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x531:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.717193:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000000d75043e/0xd5401cbabc0250ec lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x530:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.717217:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf6200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.717222:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000000d75043e/0xd5401cbabc0250ec lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x530:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.717226:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.717231:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.717235:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000000d75043e/0xd5401cbabc0250ec lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x530:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.717236:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.717246:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.717256:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000000d75043e/0xd5401cbabc0250ec lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x530:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.717269:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000d75043e/0xd5401cbabc0250ec lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x530:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.717290:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.717293:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.717296:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.717320:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514f9aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993057280:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.717388:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.717416:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.717451:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993057152:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.717469:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025108 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x533:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.717479:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.717490:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514f99f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993057408:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.717533:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.717558:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1783520935.717566:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514f9a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993057792:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.717582:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.717615:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993057280 00000100:00100000:3.0:1783520935.717620:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.717620:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993057280 00000100:00100000:1.0:1783520935.717655:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993057408 00000100:00100000:3.0:1783520935.717658:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002364928 00000100:00100000:1.0:1783520935.717658:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993057408 00000100:00100000:3.0:1783520935.717665:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002364928 00000100:00100000:1.0:1783520935.717684:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993057792 00010000:00010000:3.0:1783520935.717688:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000cbc4810a/0xd5401cbabc01f2ab lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x533:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9b0 expref: 974 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.717690:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993057792 00000100:00100000:1.0:1783520935.717721:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44096 00000100:00100000:3.0:1783520935.717729:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35686 00000100:00100000:1.0:1783520935.717731:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+28:3627:x1870152993057280:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.717745:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f1c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+974:2388:x1870153002364672:12345-192.168.204.33@tcp:103:ldlm_bl.0 00002000:00080000:1.0:1783520935.717753:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1333 count 1 00010000:00010000:3.0:1783520935.717757:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.717766:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d355441b/0xd5401cbabc01f29d lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x532:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9a2 expref: 974 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.717772:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02510f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x535:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.717790:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d355441b/0xd5401cbabc01f29d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x532:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9a2 expref: 974 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.717804:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003fe552f5/0xd5401cbabc01f2c7 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x535:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9cc expref: 974 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.717810:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x532:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.717820:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x532:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.717826:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x535:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.717834:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x535:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.717840:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02510f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x535:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.717866:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003fe552f5/0xd5401cbabc01f2c7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x535:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9cc expref: 973 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.717876:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003fe552f5/0xd5401cbabc01f2c7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x535:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9cc expref: 973 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.717885:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993057920:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.717925:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6240 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.717944:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc0250fa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x532:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.717946:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44097 00000100:00100000:1.0:1783520935.717960:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+28:3628:x1870152993057408:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.717963:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc0250fa lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x532:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.717975:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1334 count 1 00010000:00010000:3.0:1783520935.717987:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.717988:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025116 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x536:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.718016:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f1c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+971:2388:x1870153002364672:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 274us (598us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.718023:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000946a9c63/0xd5401cbabc022890 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x536:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f80 expref: 971 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.718025:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025108 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x533:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.718034:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35686 00010000:00010000:1.0:1783520935.718047:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x536:0x0].0x0 since=36s v=(0 0) 00010000:00010000:0.0:1783520935.718050:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025108 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x533:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.718052:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x536:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.718062:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025116 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x536:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.718068:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.718087:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000946a9c63/0xd5401cbabc022890 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x536:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f80 expref: 970 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.718088:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530d500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+970:2390:x1870153002364928:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 149us (474us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.718103:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35688 00010000:00010000:1.0:1783520935.718109:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000946a9c63/0xd5401cbabc022890 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x536:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f80 expref: 970 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.718123:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad1180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993058048:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.718133:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.718134:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005d2d45eb/0xd5401cbabc025101 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x534:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.718144:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005d2d45eb/0xd5401cbabc025101 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x534:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.718152:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005d2d45eb/0xd5401cbabc025101 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x534:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.718153:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6840 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.718163:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005d2d45eb/0xd5401cbabc025101 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x534:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.718174:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025108 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x533:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.718179:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.718181:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025108 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x533:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.718189:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025108 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x533:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.718197:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025108 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x533:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.718210:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520935.718261:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520935.718267:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005d2d45eb/0xd5401cbabc025101 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x534:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.718274:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635600c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+30:3626:x1870152993056768:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2423us (2561us total) trans 4294975594 rc 0/0 00010000:00010000:3.0:1783520935.718285:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf7400 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520935.718286:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44094 00010000:00010000:3.0:1783520935.718292:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005d2d45eb/0xd5401cbabc025101 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x534:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.718300:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005d2d45eb/0xd5401cbabc025101 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x534:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.718307:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514f9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+29:3626:x1870152993056640:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2370us (2544us total) trans 4294975595 rc 0/0 00000100:00100000:0.0:1783520935.718307:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44098 00010000:00010000:3.0:1783520935.718317:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005d2d45eb/0xd5401cbabc025101 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x534:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.718319:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+29:3628:x1870152993057792:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.718321:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44095 00010000:00010000:3.0:1783520935.718332:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005d2d45eb/0xd5401cbabc025101 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x534:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.718336:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1337 count 1 00010000:00010000:0.0:1783520935.718349:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003680c3ad/0xd5401cbabc02511d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x539:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.718351:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.718387:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004a76b300/0xd5401cbabc01f314 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x539:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efa04 expref: 969 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.718395:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x539:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.718398:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x539:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.718400:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514f99180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993057536:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.718404:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003680c3ad/0xd5401cbabc02511d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x539:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.718405:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025108 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x533:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.718421:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.718425:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025108 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x533:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.718431:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025108 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x533:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.718434:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.718441:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004a76b300/0xd5401cbabc01f314 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x539:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa04 expref: 969 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.718445:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025108 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x533:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.718457:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025108 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x533:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.718460:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004a76b300/0xd5401cbabc01f314 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x539:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa04 expref: 969 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.718465:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993057536 00000100:00100000:1.0:1783520935.718471:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993057536 00010000:00010000:3.0:1783520935.718479:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.718482:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514f99c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993058176:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.718483:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.718488:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.718521:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44099 00000100:00100000:1.0:1783520935.718532:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+29:3625:x1870152993057536:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.718539:0:8410:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d355441b/0xd5401cbabc01f29d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x532:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9a2 expref: 969 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.718548:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124700c0 going to sleep for 11 seconds 00002000:00080000:1.0:1783520935.718549:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1335 count 1 00000100:00100000:3.0:1783520935.718558:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad3b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993056896:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.718565:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025124 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x537:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.718571:0:25736:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002365184 00000100:00100000:0.0:1783520935.718577:0:25736:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002365184 00010000:00010000:3.0:1783520935.718580:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc0250fa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x532:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.718582:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ee8c330b/0xd5401cbabc01f2e3 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x537:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9e8 expref: 968 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.718604:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x537:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.718605:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc0250fa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x532:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.718611:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x537:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.718611:0:25736:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 9624 00010000:00010000:1.0:1783520935.718621:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025124 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x537:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.718623:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc0250fa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x532:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.718626:0:25736:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_rdpg00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+18:251787:x1870153002365184:12345-192.168.204.33@tcp:35:dd.0 00010000:00010000:3.0:1783520935.718648:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc0250fa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x532:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.718659:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc0250fa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x532:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.718670:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ee8c330b/0xd5401cbabc01f2e3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x537:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9e8 expref: 968 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.718675:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.718686:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ee8c330b/0xd5401cbabc01f2e3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x537:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9e8 expref: 968 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.718699:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad0e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993058304:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.718736:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6f00 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.718764:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1783520935.718773:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514f99880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993057664:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.718781:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+30:3628:x1870152993056512:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3111us (3255us total) trans 4294975596 rc 0/0 00000100:00100000:3.0:1783520935.718791:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44093 00000100:00100000:1.0:1783520935.718806:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000001:00080000:0.0:1783520935.718814:0:25736:0:(tgt_lastrcvd.c:2117:tgt_txn_stop_cb()) More than one transaction 8589957890 00000100:00100000:0.0:1783520935.718889:0:25736:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt_rdpg00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+20:251787:x1870153002365184:12345-192.168.204.33@tcp:35:dd.0 Request processed in 266us (758us total) trans 8589957891 rc 0/0 00000100:00100000:0.0:1783520935.718898:0:25736:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 9624 00000020:00080000:0.0:1783520935.718902:0:25736:0:(tgt_main.c:784:tgt_ses_key_exit()) total 2 transactions per RPC 00010000:00010000:0.0:1783520935.718925:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.718931:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc0250fa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x532:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.718939:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.718944:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc0250fa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x532:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.718953:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc0250fa lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x532:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.718967:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc0250fa lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x532:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.718975:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc0250fa lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x532:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.718995:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.718998:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.719001:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.719013:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993057664 00000100:00100000:0.0:1783520935.719018:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993057664 00000100:00100000:0.0:1783520935.719050:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44100 00000100:00100000:0.0:1783520935.719059:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+30:3625:x1870152993057664:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.719072:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1336 count 1 00000100:00100000:1.0:1783520935.719082:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad1180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993058048:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.719084:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc02512b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x538:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.719099:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000354e5f9a/0xd5401cbabc01f30d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x538:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9f6 expref: 968 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.719106:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x538:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.719107:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025116 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x536:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.719109:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x538:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.719120:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc02512b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x538:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.719143:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000354e5f9a/0xd5401cbabc01f30d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x538:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9f6 expref: 968 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.719152:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993057920:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.719153:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000354e5f9a/0xd5401cbabc01f30d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x538:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9f6 expref: 968 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.719168:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514f9b800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993058432:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.719178:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02510f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x535:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.719193:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.719214:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.719229:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000004:00080000:0.0:1783520935.719239:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514f9b100/2, rc 0, transno 4294975595 00000004:00080000:3.0:1783520935.719249:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514f9bb80/2, rc 0, transno 4294975596 00000100:00100000:0.0:1783520935.719253:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514f9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993056640:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.719259:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514f9bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993056512:0@lo:6:osp-syn-1-0.0 00000004:00080000:0.0:1783520935.719273:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514f99500/2, rc 0, transno 4294975594 00000100:00100000:0.0:1783520935.719280:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514f99500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993056768:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.719308:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.719339:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514f98700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993058560:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.719374:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514f98a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993058816:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.719402:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002365312 00000100:00100000:0.0:1783520935.719402:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.719408:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002365312 00000100:00100000:0.0:1783520935.719420:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1783520935.719427:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514f98380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993058688:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.719428:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.719440:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003fe552f5/0xd5401cbabc01f2c7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x535:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9cc expref: 970 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.719455:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.719487:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35690 00000100:00100000:1.0:1783520935.719494:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993058816 00000100:00100000:0.0:1783520935.719514:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993058560 00000100:00100000:3.0:1783520935.719516:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+970:2388:x1870153002365312:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.719516:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993058816 00000100:00100000:0.0:1783520935.719520:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993058560 00010000:00010000:3.0:1783520935.719532:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.719544:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993058688 00000100:00100000:0.0:1783520935.719548:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993058688 00000100:00100000:1.0:1783520935.719561:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44101 00000100:00100000:1.0:1783520935.719577:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+33:3628:x1870152993058560:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.719581:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44102 00000100:00100000:0.0:1783520935.719595:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514f98e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+33:3626:x1870152993058816:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.719596:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1338 count 1 00010000:00010000:3.0:1783520935.719603:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003fe552f5/0xd5401cbabc01f2c7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x535:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9cc expref: 969 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.719613:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc025132 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x53a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.719615:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1340 count 1 00010000:00010000:3.0:1783520935.719619:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003fe552f5/0xd5401cbabc01f2c7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x535:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9cc expref: 969 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.719632:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc025139 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x53c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.719645:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000dd15cbda/0xd5401cbabc01f31b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x53a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efa12 expref: 969 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.719654:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003fe552f5/0xd5401cbabc01f2c7 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x535:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9cc expref: 969 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.719662:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000cff5932a/0xd5401cbabc01f329 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x53c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efa2e expref: 968 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.719665:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x53a:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.719674:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x53a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.719680:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x535:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.719683:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc025132 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x53a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.719689:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x535:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.719690:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x53c:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.719700:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02510f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x535:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.719701:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x53c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.719710:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc025139 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x53c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.719715:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000dd15cbda/0xd5401cbabc01f31b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x53a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa12 expref: 967 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.719716:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02510f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x535:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.719740:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000dd15cbda/0xd5401cbabc01f31b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x53a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa12 expref: 967 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.719745:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000cff5932a/0xd5401cbabc01f329 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x53c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa2e expref: 967 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.719756:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.719764:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000cff5932a/0xd5401cbabc01f329 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x53c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa2e expref: 967 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.719766:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad0380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993058944:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.719781:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+967:2388:x1870153002365312:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 281us (552us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.719801:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35690 00000100:00100000:1.0:1783520935.719806:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6240 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.719811:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943512427800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993059072:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.719834:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44103 00000100:00100000:0.0:1783520935.719841:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470f00 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.719848:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943512426300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+33:3626:x1870152993058688:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.719862:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1339 count 1 00010000:00010000:1.0:1783520935.719875:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025140 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x53b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.719876:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025116 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x536:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.719896:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025116 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x536:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.719897:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a987e62f/0xd5401cbabc01f322 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x53b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efa20 expref: 966 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.719919:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x53b:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.719922:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025116 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x536:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.719924:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x53b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.719932:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025140 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x53b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.719945:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025116 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x536:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.719965:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520935.719971:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a987e62f/0xd5401cbabc01f322 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x53b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa20 expref: 966 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.719981:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a987e62f/0xd5401cbabc01f322 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x53b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa20 expref: 966 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.719993:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad3800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993059200:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.719994:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520935.720002:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02510f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x535:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.720011:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.720015:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02510f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x535:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.720025:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02510f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x535:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.720026:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6840 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.720043:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02510f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x535:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.720044:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.720051:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025116 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x536:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.720053:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02510f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x535:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.720060:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+34:3628:x1870152993057408:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2101us (2524us total) trans 4294975598 rc 0/0 00010000:00010000:1.0:1783520935.720077:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6400 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520935.720080:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44097 00010000:00010000:1.0:1783520935.720083:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025116 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x536:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.720105:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025116 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x536:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.720121:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025116 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x536:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.720130:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025116 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x536:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.720132:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514f99f80/2, rc 0, transno 4294975598 00010000:00010000:3.0:1783520935.720136:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.720141:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.720146:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.720148:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514f99f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993057408:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.720156:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.720160:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.720164:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520935.720176:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514f9aa00/2, rc 0, transno 4294975597 00000100:00100000:1.0:1783520935.720178:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002365568 00000100:00100000:3.0:1783520935.720185:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514f9aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993057280:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.720185:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002365568 00000100:00100000:3.0:1783520935.720201:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943512427100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993059328:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.720210:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943512426680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993059456:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.720211:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004a76b300/0xd5401cbabc01f314 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x539:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa04 expref: 967 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.720236:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.720243:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002365696 00000100:00100000:1.0:1783520935.720248:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002365696 00010000:00010000:1.0:1783520935.720264:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ee8c330b/0xd5401cbabc01f2e3 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x537:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9e8 expref: 968 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.720271:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35691 00000100:00100000:0.0:1783520935.720286:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517245880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+968:2388:x1870153002365568:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.720296:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002365824 00010000:00010000:0.0:1783520935.720298:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.720301:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002365824 00010000:00010000:0.0:1783520935.720307:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004a76b300/0xd5401cbabc01f314 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x539:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa04 expref: 969 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.720320:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000354e5f9a/0xd5401cbabc01f30d lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x538:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9f6 expref: 969 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.720322:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004a76b300/0xd5401cbabc01f314 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x539:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa04 expref: 969 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.720326:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.720354:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x539:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.720379:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993059328 00000100:00100000:3.0:1783520935.720383:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993059328 00010000:00010000:0.0:1783520935.720390:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x539:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.720393:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000003680c3ad/0xd5401cbabc02511d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x539:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.720396:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35692 00010000:00010000:0.0:1783520935.720404:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003680c3ad/0xd5401cbabc02511d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x539:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.720406:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993059456 00000100:00100000:3.0:1783520935.720411:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993059456 00000100:00100000:1.0:1783520935.720413:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517246680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+969:2388:x1870153002365696:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.720420:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.720428:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.720437:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ee8c330b/0xd5401cbabc01f2e3 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x537:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9e8 expref: 968 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.720438:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44104 00000100:00100000:0.0:1783520935.720448:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517245880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+968:2388:x1870153002365568:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 165us (359us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.720453:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+35:3627:x1870152993059328:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.720466:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35691 00010000:00010000:1.0:1783520935.720467:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ee8c330b/0xd5401cbabc01f2e3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x537:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9e8 expref: 968 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520935.720472:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1341 count 1 00000100:00100000:0.0:1783520935.720485:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35693 00010000:00010000:3.0:1783520935.720489:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025147 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x53d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.720491:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x537:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.720509:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f1450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+967:2387:x1870153002365824:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.720516:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x537:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.720523:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.720527:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008f02dfd3/0xd5401cbabc01f345 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x53d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efa3c expref: 967 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.720527:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025124 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x537:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.720531:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000354e5f9a/0xd5401cbabc01f30d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x538:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9f6 expref: 967 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.720553:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x53d:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.720556:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025124 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x537:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.720562:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x53d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.720572:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025147 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x53d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.720578:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000354e5f9a/0xd5401cbabc01f30d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x538:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9f6 expref: 967 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.720591:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.720609:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000354e5f9a/0xd5401cbabc01f30d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x538:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9f6 expref: 967 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.720618:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517246680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+967:2388:x1870153002365696:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 208us (439us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.720619:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008f02dfd3/0xd5401cbabc01f345 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x53d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa3c expref: 967 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.720629:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x538:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.720638:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x538:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.720639:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35692 00010000:00010000:0.0:1783520935.720648:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc02512b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x538:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.720657:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008f02dfd3/0xd5401cbabc01f345 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x53d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa3c expref: 965 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.720660:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc02512b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x538:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.720684:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530ed80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993059584:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.720696:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.720710:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc02512b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x538:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.720721:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f1450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+965:2387:x1870153002365824:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 225us (445us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.720737:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c8729c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.720737:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35693 00010000:00010000:1.0:1783520935.720738:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc02512b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x538:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.720753:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc02512b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x538:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.720772:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514f9a680/2, rc 0, transno 4294975599 00010000:00010000:0.0:1783520935.720773:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.720777:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc02512b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x538:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.720779:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000003680c3ad/0xd5401cbabc02511d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x539:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.720788:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514f9a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993057792:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.720806:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf7a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.720810:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.720813:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000003680c3ad/0xd5401cbabc02511d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x539:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.720815:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.720823:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc02512b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x538:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.720851:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf6a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.720852:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000003680c3ad/0xd5401cbabc02511d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x539:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.720858:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc02512b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x538:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.720871:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000003680c3ad/0xd5401cbabc02511d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x539:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.720882:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc02512b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x538:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.720890:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003680c3ad/0xd5401cbabc02511d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x539:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.720897:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+36:3625:x1870152993057664:12345-0@lo:6:osp-syn-1-0.0 Request processed in 1838us (2090us total) trans 4294975600 rc 0/0 00010000:00010000:3.0:1783520935.720910:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc02512b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x538:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.720919:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44100 00010000:00010000:0.0:1783520935.720922:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.720925:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.720930:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc02512b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x538:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.720930:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.720958:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.720962:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520935.720963:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514f99880/2, rc 0, transno 4294975600 00010000:00010000:3.0:1783520935.720966:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.720970:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1783520935.720977:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514f99880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993057664:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.720979:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943512427480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993059840:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.721014:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.721038:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993059840 00000100:00100000:1.0:1783520935.721043:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993059840 00000100:00100000:1.0:1783520935.721079:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44106 00000100:00100000:1.0:1783520935.721089:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943512424a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+36:3626:x1870152993059840:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.721102:0:35276:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ee8c330b/0xd5401cbabc01f2e3 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x537:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859ef9e8 expref: 964 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.721103:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1343 count 1 00010000:00010000:1.0:1783520935.721117:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025155 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x53f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.721118:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad0e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993058304:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.721134:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000824f0e0b/0xd5401cbabc01f353 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x53f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efa58 expref: 963 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.721135:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025124 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x537:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.721160:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025124 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x537:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.721162:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x53f:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.721170:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x53f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.721178:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025155 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x53f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.721187:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025124 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x537:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.721199:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025124 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x537:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.721205:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000824f0e0b/0xd5401cbabc01f353 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x53f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa58 expref: 963 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.721209:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025124 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x537:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.721213:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943512427b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993059968:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.721225:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000824f0e0b/0xd5401cbabc01f353 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x53f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa58 expref: 963 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.721233:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.721240:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993060096:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.721285:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6300 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.721305:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+37:3625:x1870152993057536:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2775us (2871us total) trans 4294975601 rc 0/0 00000100:00100000:1.0:1783520935.721315:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad3800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993059200:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.721316:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44099 00000100:00100000:3.0:1783520935.721330:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.721338:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025140 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x53b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.721341:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993059968 00000100:00100000:0.0:1783520935.721347:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993059968 00000100:00100000:1.0:1783520935.721399:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad0380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993058944:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.721417:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc025132 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x53a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.721418:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44107 00010000:00010000:3.0:1783520935.721429:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520935.721431:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+37:3626:x1870152993059968:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.721436:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025124 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x537:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.721445:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002365952 00002000:00080000:0.0:1783520935.721445:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1344 count 1 00010000:00010000:3.0:1783520935.721447:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7800 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520935.721451:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002365952 00010000:00010000:3.0:1783520935.721454:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025124 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x537:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.721462:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df5f0644/0xd5401cbabc02515c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x540:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.721471:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025124 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x537:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.721474:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000dd15cbda/0xd5401cbabc01f31b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x53a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa12 expref: 964 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.721478:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e8a1c07d/0xd5401cbabc01f376 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x540:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efa66 expref: 964 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.721511:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x540:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.721516:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x540:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.721519:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025124 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x537:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.721521:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df5f0644/0xd5401cbabc02515c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x540:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.721527:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002366080 00000100:00100000:1.0:1783520935.721533:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002366080 00010000:00010000:3.0:1783520935.721546:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025124 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x537:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.721552:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000cff5932a/0xd5401cbabc01f329 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x53c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa2e expref: 966 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.721563:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e8a1c07d/0xd5401cbabc01f376 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x540:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa66 expref: 966 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.721569:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.721573:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.721577:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.721583:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e8a1c07d/0xd5401cbabc01f376 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x540:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa66 expref: 966 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.721595:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35695 00000100:00100000:0.0:1783520935.721609:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943512425180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993060224:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.721610:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517245880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+966:2387:x1870153002366208:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.721626:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.721635:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a987e62f/0xd5401cbabc01f322 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x53b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa20 expref: 966 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.721651:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a987e62f/0xd5401cbabc01f322 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x53b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa20 expref: 966 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.721660:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470b40 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.721683:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a987e62f/0xd5401cbabc01f322 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x53b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa20 expref: 965 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.721699:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x53b:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.721705:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35696 00010000:00010000:1.0:1783520935.721708:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x53b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.721718:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025140 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x53b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.721720:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517246680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+964:2387:x1870153002366080:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.721735:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025140 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x53b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.721736:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.721747:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000cff5932a/0xd5401cbabc01f329 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x53c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa2e expref: 964 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.721748:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc025132 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x53a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.721762:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.721776:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000cff5932a/0xd5401cbabc01f329 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x53c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa2e expref: 964 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.721791:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517245880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+964:2387:x1870153002366208:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 182us (371us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.721798:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x53c:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.721800:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35695 00010000:00010000:0.0:1783520935.721804:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x53c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.721811:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc025139 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x53c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.721812:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc025132 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x53a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.721829:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025140 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x53b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.721837:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc025132 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x53a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.721855:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025140 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x53b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.721856:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc025139 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x53c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.721857:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc025132 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x53a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.721881:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025140 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x53b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.721891:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025140 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x53b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.721891:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.721894:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.721913:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.721922:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517246680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+962:2387:x1870153002366080:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 204us (464us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.721937:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35696 00010000:00010000:0.0:1783520935.721960:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.721968:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025140 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x53b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.721978:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.721984:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025140 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x53b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.721995:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025140 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x53b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.722000:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+39:3628:x1870152993058560:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2426us (2572us total) trans 4294975602 rc 0/0 00000100:00100000:3.0:1783520935.722013:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44101 00010000:00010000:0.0:1783520935.722016:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025140 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x53b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.722020:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943512426300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+39:3626:x1870152993058688:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2174us (2565us total) trans 4294975603 rc 0/0 00010000:00010000:0.0:1783520935.722035:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025140 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x53b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.722037:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44103 00010000:00010000:0.0:1783520935.722064:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.722067:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.722070:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520935.722098:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514f98700/2, rc 0, transno 4294975602 00000004:00080000:1.0:1783520935.722129:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514f99180/2, rc 0, transno 4294975601 00000100:00100000:1.0:1783520935.722137:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514f99180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993057536:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.722164:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943512425c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993060608:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.722177:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514f98700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993058560:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.722192:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943512426a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993060480:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.722202:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.722233:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.722288:0:12734:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cff5932a/0xd5401cbabc01f329 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x53c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa2e expref: 961 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.722301:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943512427800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993059072:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.722314:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc025139 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x53c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.722322:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc025139 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x53c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.722329:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc025139 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x53c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.722338:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc025139 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x53c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.722345:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc025139 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x53c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.722425:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.722437:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993060352 00000100:00100000:1.0:1783520935.722443:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993060352 00010000:00010000:3.0:1783520935.722466:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.722472:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc025139 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x53c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.722473:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993060608 00000100:00100000:1.0:1783520935.722476:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993060608 00010000:00010000:3.0:1783520935.722485:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf6e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.722489:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc025139 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x53c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.722494:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993060480 00000100:00100000:1.0:1783520935.722513:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993060480 00010000:00010000:3.0:1783520935.722521:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc025139 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x53c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.722537:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc025139 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x53c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.722542:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44108 00000100:00100000:0.0:1783520935.722544:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514f98e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+41:3626:x1870152993058816:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2951us (3141us total) trans 4294975604 rc 0/0 00000100:00100000:1.0:1783520935.722558:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517244380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+41:3627:x1870152993060352:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.722559:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc025139 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x53c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.722563:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44102 00002000:00080000:1.0:1783520935.722579:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1345 count 1 00010000:00010000:3.0:1783520935.722592:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.722593:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44109 00010000:00010000:1.0:1783520935.722596:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025163 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x541:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.722597:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.722606:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+40:3625:x1870152993060608:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.722619:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a9d1a4f8/0xd5401cbabc01f37d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x541:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efa74 expref: 961 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.722623:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1347 count 1 00010000:00010000:0.0:1783520935.722639:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc02516a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x543:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.722643:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x541:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.722651:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x541:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.722657:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025163 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x541:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.722660:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000be4b4dbc/0xd5401cbabc01f36f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x543:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efa89 expref: 961 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.722680:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x543:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.722686:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.722690:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x543:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.722695:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a9d1a4f8/0xd5401cbabc01f37d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x541:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa74 expref: 962 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.722698:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc02516a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x543:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.722702:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44110 00000100:00100000:3.0:1783520935.722715:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+40:3628:x1870152993060480:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.722723:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a9d1a4f8/0xd5401cbabc01f37d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x541:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa74 expref: 963 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520935.722733:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1346 count 1 00000100:00100000:1.0:1783520935.722740:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad3800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993060736:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.722742:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000be4b4dbc/0xd5401cbabc01f36f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x543:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa89 expref: 963 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.722748:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025171 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x542:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.722773:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000be4b4dbc/0xd5401cbabc01f36f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x543:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa89 expref: 963 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.722775:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000035406f96/0xd5401cbabc01f384 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x542:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efa82 expref: 963 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.722793:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6840 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.722798:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x542:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.722803:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x542:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.722805:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943512427800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993060864:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.722808:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025171 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x542:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.722835:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35698 00010000:00010000:3.0:1783520935.722839:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000035406f96/0xd5401cbabc01f384 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x542:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa82 expref: 963 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.722839:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470240 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.722854:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517245500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+963:2387:x1870153002366336:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.722864:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000035406f96/0xd5401cbabc01f384 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x542:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa82 expref: 963 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.722868:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000004:00080000:0.0:1783520935.722871:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514f98a80/2, rc 0, transno 4294975604 00000100:00100000:3.0:1783520935.722878:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530ea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993060992:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.722879:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008f02dfd3/0xd5401cbabc01f345 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x53d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa3c expref: 963 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.722885:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514f98a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993058816:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.722906:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008f02dfd3/0xd5401cbabc01f345 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x53d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa3c expref: 962 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.722908:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872b40 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.722929:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x53d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.722934:0:35275:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008f02dfd3/0xd5401cbabc01f345 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x53d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa3c expref: 962 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.722937:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x53d:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.722944:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943512425880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993061120:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.722948:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025147 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x53d:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.722961:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530ed80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993059584:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.722964:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025147 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x53d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.722975:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025147 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x53d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.722983:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.722984:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025147 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x53d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.722988:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.722998:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025147 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x53d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.723015:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517245500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+961:2387:x1870153002366336:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 164us (516us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.723018:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025147 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x53d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.723028:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993061120 00000100:00100000:1.0:1783520935.723033:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35698 00000100:00100000:0.0:1783520935.723037:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993061120 00010000:00010000:3.0:1783520935.723046:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025147 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x53d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.723060:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.723076:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44111 00000100:00100000:1.0:1783520935.723089:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515476680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+41:3626:x1870152993061120:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.723108:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1348 count 1 00000100:00100000:0.0:1783520935.723122:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.723123:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025178 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x544:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.723150:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+42:3627:x1870152993059328:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2699us (2914us total) trans 4294975605 rc 0/0 00010000:00010000:1.0:1783520935.723160:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000aa4edb5a/0xd5401cbabc01f399 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x544:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efa90 expref: 959 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.723164:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44104 00010000:00010000:1.0:1783520935.723175:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x544:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.723179:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x544:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.723184:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025178 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x544:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.723198:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943512427100/2, rc 0, transno 4294975605 00000100:00100000:3.0:1783520935.723212:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943512427100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993059328:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.723213:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000aa4edb5a/0xd5401cbabc01f399 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x544:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa90 expref: 958 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.723231:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000aa4edb5a/0xd5401cbabc01f399 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x544:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa90 expref: 959 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.723250:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad0000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993061248:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.723265:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943512425180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993060224:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.723267:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1783520935.723281:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993061376:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.723283:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000df5f0644/0xd5401cbabc02515c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x540:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.723289:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6c00 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.723318:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.723332:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000824f0e0b/0xd5401cbabc01f353 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x53f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa58 expref: 959 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.723379:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993060096:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.723385:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.723394:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025147 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x53d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.723400:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025155 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x53f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.723409:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.723417:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025147 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x53d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.723423:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025155 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x53f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.723429:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025147 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x53d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.723452:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025155 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x53f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.723456:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025147 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x53d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.723482:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025155 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x53f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.723483:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025147 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x53d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.723516:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000df5f0644/0xd5401cbabc02515c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x540:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.723522:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025155 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x53f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.723531:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.723533:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000df5f0644/0xd5401cbabc02515c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x540:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.723535:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.723540:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.723553:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df5f0644/0xd5401cbabc02515c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x540:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.723555:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993061376 00010000:00010000:1.0:1783520935.723556:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.723564:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993061376 00010000:00010000:3.0:1783520935.723566:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000df5f0644/0xd5401cbabc02515c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x540:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.723579:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:0.0:1783520935.723601:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44112 00000100:00100000:0.0:1783520935.723614:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+43:3628:x1870152993061376:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.723634:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1349 count 1 00010000:00010000:0.0:1783520935.723649:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000013affd72/0xd5401cbabc02517f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x545:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.723659:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943512424a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+44:3626:x1870152993059840:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2569us (2641us total) trans 4294975606 rc 0/0 00000100:00100000:3.0:1783520935.723671:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+44:3626:x1870152993059968:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2241us (2339us total) trans 4294975607 rc 0/0 00010000:00010000:0.0:1783520935.723675:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f0038863/0xd5401cbabc01f38b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x545:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efa97 expref: 956 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.723678:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44106 00000100:00100000:3.0:1783520935.723690:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44107 00010000:00010000:0.0:1783520935.723701:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x545:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.723705:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x545:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.723709:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000013affd72/0xd5401cbabc02517f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x545:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.723724:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943512427b80/2, rc 0, transno 4294975607 00010000:00010000:0.0:1783520935.723732:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f0038863/0xd5401cbabc01f38b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x545:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa97 expref: 956 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.723741:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943512427b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993059968:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.723754:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f0038863/0xd5401cbabc01f38b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x545:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa97 expref: 956 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.723762:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943512427480/2, rc 0, transno 4294975606 00000100:00100000:1.0:1783520935.723773:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943512427480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993059840:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.723776:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515475880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993061504:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.723811:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1783520935.723815:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1783520935.723821:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470b40 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.723824:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993061632:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.723829:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993061760:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.723838:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.723864:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.723881:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02514e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x53e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.723895:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993061632 00010000:00010000:0.0:1783520935.723897:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02514e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x53e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.723901:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993061632 00010000:00010000:0.0:1783520935.723910:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02514e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x53e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.723918:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02514e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x53e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.723933:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44113 00010000:00010000:0.0:1783520935.723935:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.723944:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+43:3627:x1870152993061632:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.723947:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.723952:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02514e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x53e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.723958:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185000 already unused, calling callback (ffffffffc0bf9e90) 00002000:00080000:1.0:1783520935.723958:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1350 count 1 00010000:00010000:3.0:1783520935.723963:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02514e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x53e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.723974:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc025186 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x546:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.723988:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b8b4dac0/0xd5401cbabc01f392 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x546:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efaa5 expref: 956 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.723995:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x546:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.723998:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x546:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.724003:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc025186 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x546:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.724008:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02514e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x53e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.724020:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+44:3628:x1870152993059456:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3308us (3732us total) trans 4294975608 rc 0/0 00010000:00010000:3.0:1783520935.724026:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02514e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x53e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.724027:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b8b4dac0/0xd5401cbabc01f392 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x546:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efaa5 expref: 956 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.724032:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44105 00010000:00010000:3.0:1783520935.724049:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02514e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x53e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.724055:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b8b4dac0/0xd5401cbabc01f392 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x546:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efaa5 expref: 956 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.724072:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.724075:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.724076:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993061888:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.724079:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.724084:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.724119:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6540 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.724160:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993062016:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.724194:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.724394:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993062016 00000100:00100000:1.0:1783520935.724399:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993062016 00000100:00100000:1.0:1783520935.724441:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44115 00000100:00100000:1.0:1783520935.724454:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+45:3626:x1870152993062016:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.724469:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1352 count 1 00000100:00100000:0.0:1783520935.724473:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad3800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993060736:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.724485:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc025194 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x548:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.724494:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025163 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x541:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.724530:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000057858a4e/0xd5401cbabc01f3bc lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x548:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efac1 expref: 956 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.724541:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x548:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.724544:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x548:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.724549:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc025194 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x548:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.724554:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002366848 00000100:00100000:0.0:1783520935.724561:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002366848 00010000:00010000:1.0:1783520935.724574:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000057858a4e/0xd5401cbabc01f3bc lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x548:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efac1 expref: 957 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.724580:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000be4b4dbc/0xd5401cbabc01f36f lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x543:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa89 expref: 957 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.724601:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000057858a4e/0xd5401cbabc01f3bc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x548:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efac1 expref: 957 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.724601:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002366976 00000100:00100000:0.0:1783520935.724606:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002366976 00010000:00010000:0.0:1783520935.724620:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a9d1a4f8/0xd5401cbabc01f37d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x541:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa74 expref: 958 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.724625:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad3800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993062272:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.724638:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002367104 00000100:00100000:0.0:1783520935.724641:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002367104 00010000:00010000:0.0:1783520935.724652:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000035406f96/0xd5401cbabc01f384 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x542:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa82 expref: 959 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.724660:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35701 00000100:00100000:3.0:1783520935.724673:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517245880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+959:2388:x1870153002366848:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.724684:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.724684:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35702 00000100:00100000:1.0:1783520935.724688:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6cc0 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.724695:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000be4b4dbc/0xd5401cbabc01f36f lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x543:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa89 expref: 959 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.724695:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517246680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+959:2389:x1870153002366976:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.724706:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.724712:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a9d1a4f8/0xd5401cbabc01f37d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x541:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa74 expref: 959 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.724722:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad0000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993061248:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.724729:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a9d1a4f8/0xd5401cbabc01f37d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x541:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa74 expref: 959 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.724739:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025178 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x544:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.724750:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a9d1a4f8/0xd5401cbabc01f37d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x541:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa74 expref: 959 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.724762:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x541:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.724767:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x541:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.724773:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025163 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x541:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.724789:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025163 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x541:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.724822:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.724841:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025163 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x541:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.724848:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517246680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+958:2389:x1870153002366976:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 156us (287us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.724864:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025163 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x541:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.724868:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35702 00010000:00010000:3.0:1783520935.724883:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000be4b4dbc/0xd5401cbabc01f36f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x543:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa89 expref: 957 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.724887:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025163 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x541:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.724906:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.724909:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025163 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x541:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.724913:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x543:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.724917:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x543:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.724920:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc02516a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x543:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.724923:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.724930:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc02516a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x543:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.724949:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.724977:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517245880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+957:2388:x1870153002366848:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 307us (452us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.724987:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35701 00000100:00100000:1.0:1783520935.725040:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517244380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+46:3627:x1870152993060352:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2486us (2867us total) trans 4294975609 rc 0/0 00000100:00100000:1.0:1783520935.725053:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44108 00010000:00010000:0.0:1783520935.725073:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.725077:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025163 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x541:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.725087:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.725092:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025163 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x541:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.725099:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943512424700/2, rc 0, transno 4294975609 00010000:00010000:0.0:1783520935.725107:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025163 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x541:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.725115:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943512424700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993060352:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.725133:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025163 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x541:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.725146:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993061888:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.725152:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025163 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x541:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.725166:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc025186 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x546:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.725176:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.725179:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.725183:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.725186:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025178 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x544:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.725191:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025178 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x544:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.725199:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025178 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x544:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.725213:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515475880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993061504:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.725233:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000013affd72/0xd5401cbabc02517f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x545:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.725240:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025178 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x544:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.725261:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.725270:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.725276:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025178 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x544:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.725283:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.725288:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025178 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x544:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.725301:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025178 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x544:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.725316:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025178 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x544:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.725324:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025178 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x544:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.725339:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.725342:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.725346:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.725350:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515476680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+46:3626:x1870152993061120:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2262us (2362us total) trans 4294975610 rc 0/0 00000100:00100000:1.0:1783520935.725389:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44111 00000100:00100000:1.0:1783520935.725409:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993062400 00000100:00100000:1.0:1783520935.725414:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993062400 00000100:00100000:1.0:1783520935.725444:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44116 00000100:00100000:1.0:1783520935.725457:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517245500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+46:3628:x1870152993062400:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.725476:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1353 count 1 00010000:00010000:1.0:1783520935.725490:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02519b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x549:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.725508:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.725513:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000cb1703c4/0xd5401cbabc01f3ca lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x549:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efacf expref: 953 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.725515:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025171 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x542:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.725533:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x549:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.725542:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x549:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.725543:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.725548:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025171 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x542:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.725550:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02519b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x549:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.725562:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025171 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x542:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.725573:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025171 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x542:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.725581:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000cb1703c4/0xd5401cbabc01f3ca lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x549:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efacf expref: 953 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.725584:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025171 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x542:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.725617:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000cb1703c4/0xd5401cbabc01f3ca lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x549:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efacf expref: 953 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.725620:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.725625:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.725630:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993062528:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.725630:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520935.725632:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943512426a00/2, rc 0, transno 4294975611 00000100:00100000:3.0:1783520935.725643:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943512426a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993060480:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.725668:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.725684:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6540 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.725694:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1783520935.725706:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993062656:0@lo:6:osp-syn-1-0.0 00000004:00080000:1.0:1783520935.725718:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943512425880/2, rc 0, transno 4294975610 00000100:00100000:1.0:1783520935.725735:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943512425880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993061120:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.725748:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002367360 00000100:00100000:0.0:1783520935.725754:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002367360 00000100:00100000:3.0:1783520935.725757:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.725777:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f0038863/0xd5401cbabc01f38b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x545:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa97 expref: 953 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.725796:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35705 00000100:00100000:0.0:1783520935.725803:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515475880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+953:2388:x1870153002367360:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.725811:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.725814:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f0038863/0xd5401cbabc01f38b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x545:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa97 expref: 953 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.725823:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f0038863/0xd5401cbabc01f38b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x545:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa97 expref: 953 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.725835:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f0038863/0xd5401cbabc01f38b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x545:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efa97 expref: 953 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.725836:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520935.725844:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.725845:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc02516a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x543:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.725854:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x545:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.725857:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf6600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.725862:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x545:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.725863:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc02516a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x543:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.725874:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000013affd72/0xd5401cbabc02517f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x545:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.725877:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc02516a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x543:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.725888:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.725901:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000013affd72/0xd5401cbabc02517f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x545:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.725903:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc02516a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x543:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.725921:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc02516a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x543:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.725937:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.725941:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.725944:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.725945:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.725947:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.725966:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515475880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+952:2388:x1870153002367360:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 162us (299us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.725976:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35705 00000100:00100000:3.0:1783520935.725977:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002367488 00000100:00100000:3.0:1783520935.725983:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002367488 00000100:00100000:0.0:1783520935.725991:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002367616 00000100:00100000:0.0:1783520935.725996:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002367616 00010000:00010000:3.0:1783520935.726004:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b8b4dac0/0xd5401cbabc01f392 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x546:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efaa5 expref: 953 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.726014:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005eb37d24/0xd5401cbabc01f3ae lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x547:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efab3 expref: 953 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.726024:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002367744 00000100:00100000:1.0:1783520935.726031:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002367744 00000100:00100000:3.0:1783520935.726040:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35706 00010000:00010000:1.0:1783520935.726054:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000057858a4e/0xd5401cbabc01f3bc lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x548:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efac1 expref: 954 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.726054:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35707 00000100:00100000:3.0:1783520935.726056:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+954:2390:x1870153002367488:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.726070:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f3050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+954:2388:x1870153002367616:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.726073:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.726083:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b8b4dac0/0xd5401cbabc01f392 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x546:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efaa5 expref: 954 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.726085:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.726095:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005eb37d24/0xd5401cbabc01f3ae lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x547:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efab3 expref: 954 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.726097:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35708 00010000:00010000:3.0:1783520935.726109:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b8b4dac0/0xd5401cbabc01f392 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x546:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efaa5 expref: 954 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.726114:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+954:2390:x1870153002367744:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.726120:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005eb37d24/0xd5401cbabc01f3ae lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x547:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efab3 expref: 954 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.726130:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.726141:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000057858a4e/0xd5401cbabc01f3bc lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x548:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efac1 expref: 954 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.726144:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x547:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.726145:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b8b4dac0/0xd5401cbabc01f392 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x546:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efaa5 expref: 954 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.726152:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x547:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.726163:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02518d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x547:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.726172:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000057858a4e/0xd5401cbabc01f3bc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x548:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efac1 expref: 954 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.726174:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02518d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x547:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.726176:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x546:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.726186:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x546:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.726197:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc025186 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x546:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.726200:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.726210:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc025186 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x546:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.726212:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x548:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.726218:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x548:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.726223:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc025194 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x548:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.726230:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f3050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+953:2388:x1870153002367616:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 162us (346us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.726237:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc025194 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x548:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.726243:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35707 00010000:00010000:1.0:1783520935.726260:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.726270:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000013affd72/0xd5401cbabc02517f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x545:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.726276:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000013affd72/0xd5401cbabc02517f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x545:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.726286:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000013affd72/0xd5401cbabc02517f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x545:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.726292:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+952:2390:x1870153002367744:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 180us (348us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.726294:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000013affd72/0xd5401cbabc02517f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x545:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.726309:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35708 00010000:00010000:0.0:1783520935.726324:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.726333:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.726342:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc025186 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x546:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.726354:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc025186 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x546:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.726418:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc025186 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x546:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.726427:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc025186 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x546:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.726436:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+48:3628:x1870152993061376:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2826us (3120us total) trans 4294975613 rc 0/0 00010000:00010000:1.0:1783520935.726444:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:0.0:1783520935.726446:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44112 00000100:00100000:3.0:1783520935.726447:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+951:2390:x1870153002367488:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 330us (541us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.726470:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.726476:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc025186 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x546:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.726491:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.726509:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc025186 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x546:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.726531:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc025186 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x546:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.726545:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+48:3627:x1870152993061632:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2602us (2680us total) trans 4294975614 rc 0/0 00010000:00010000:0.0:1783520935.726546:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc025186 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x546:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.726563:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44113 00010000:00010000:0.0:1783520935.726569:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc025186 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x546:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.726588:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.726592:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.726597:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.726605:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993062528:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.726614:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993062784:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.726620:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02519b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x549:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.726646:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.726659:0:20415:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000057858a4e/0xd5401cbabc01f3bc lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x548:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efac1 expref: 951 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.726674:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad3800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993062272:192.168.204.33@tcp:104: 00000004:00080000:0.0:1783520935.726681:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943512425c00/2, rc 0, transno 4294975612 00010000:00010000:1.0:1783520935.726691:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc025194 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x548:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.726694:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943512425c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993060608:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.726708:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc025194 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x548:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.726724:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc025194 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x548:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.726737:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc025194 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x548:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.726749:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc025194 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x548:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.726788:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.726810:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.726816:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc025194 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x548:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.726834:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.726839:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc025194 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x548:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.726861:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc025194 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x548:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.726868:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+49:3626:x1870152993062016:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2416us (2671us total) trans 4294975615 rc 0/0 00000100:00100000:1.0:1783520935.726885:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44115 00010000:00010000:0.0:1783520935.726891:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc025194 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x548:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.726900:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc025194 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x548:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.726922:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.726925:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.726929:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.726938:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993063168:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.726963:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.726988:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993063168 00000100:00100000:0.0:1783520935.726994:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993063168 00000004:00080000:1.0:1783520935.726996:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351530df80/2, rc 0, transno 4294975615 00000100:00100000:1.0:1783520935.727007:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993062016:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.727027:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44119 00000100:00100000:0.0:1783520935.727036:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+49:3626:x1870152993063168:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.727052:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1356 count 1 00010000:00010000:1.0:1783520935.727057:0:35276:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005eb37d24/0xd5401cbabc01f3ae lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x547:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efab3 expref: 950 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.727069:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0251b0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x54c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.727071:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517244700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993062144:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.727088:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000fd320b0d/0xd5401cbabc01f3df lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x54c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efaf9 expref: 949 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.727089:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02518d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x547:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.727102:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x54c:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.727107:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x54c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.727108:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02518d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x547:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.727111:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0251b0 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x54c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.727125:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02518d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x547:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.727134:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02518d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x547:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.727139:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000fd320b0d/0xd5401cbabc01f3df lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x54c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efaf9 expref: 949 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.727160:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000fd320b0d/0xd5401cbabc01f3df lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x54c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efaf9 expref: 949 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.727162:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02518d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x547:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.727172:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35706 00000100:00100000:0.0:1783520935.727177:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515475500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993063424:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.727191:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.727215:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470900 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.727281:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520935.727281:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515474e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+50:3626:x1870152993061760:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3132us (3208us total) trans 4294975616 rc 0/0 00010000:00010000:3.0:1783520935.727289:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02518d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x547:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.727294:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44114 00010000:00010000:3.0:1783520935.727314:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.727319:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02518d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x547:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.727330:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351530f100/2, rc 0, transno 4294975616 00000100:00100000:1.0:1783520935.727344:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993061760:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.727389:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02518d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x547:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.727411:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02518d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x547:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.727430:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02518d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x547:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.727514:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02519b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x549:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.727533:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02519b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x549:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.727543:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02519b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x549:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.727551:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02519b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x549:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.727567:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.727642:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517245500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+50:3628:x1870152993062400:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2186us (2345us total) trans 4294975617 rc 0/0 00000100:00100000:1.0:1783520935.727652:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44116 00010000:00010000:3.0:1783520935.727706:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.727711:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.727715:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.727718:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517247b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993062912:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.727730:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002368000 00000100:00100000:3.0:1783520935.727736:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515475180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993063552:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.727737:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002368000 00010000:00010000:0.0:1783520935.727738:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000009eb77304/0xd5401cbabc0251a2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x54a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.727763:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007f83bd80/0xd5401cbabc01f3d1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x54a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efadd expref: 948 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.727773:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.727791:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35710 00000100:00100000:1.0:1783520935.727803:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517245180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+948:2387:x1870153002368000:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000004:00080000:3.0:1783520935.727811:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351530d880/2, rc 0, transno 4294975617 00010000:00010000:1.0:1783520935.727813:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.727822:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007f83bd80/0xd5401cbabc01f3d1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x54a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efadd expref: 948 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.727827:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993062400:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.727844:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007f83bd80/0xd5401cbabc01f3d1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x54a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efadd expref: 948 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.727849:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351530ed80/2, rc 0, transno 4294975613 00000100:00100000:0.0:1783520935.727866:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515475c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993063680:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.727870:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007f83bd80/0xd5401cbabc01f3d1 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x54a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efadd expref: 948 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.727884:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x54a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.727890:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x54a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.727896:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000009eb77304/0xd5401cbabc0251a2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x54a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.727912:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.727920:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000009eb77304/0xd5401cbabc0251a2 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x54a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.727931:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993063680 00000100:00100000:0.0:1783520935.727936:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993063680 00000100:00100000:3.0:1783520935.727958:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993061376:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.727967:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44121 00010000:00010000:1.0:1783520935.727971:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.727980:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515475880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+51:3625:x1870152993063680:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.727982:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1783520935.727995:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993063296:0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.727997:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1359 count 1 00000100:00100000:1.0:1783520935.728004:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517245180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+947:2387:x1870153002368000:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 202us (301us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.728013:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35710 00010000:00010000:0.0:1783520935.728017:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0251be lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x54f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.728031:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000071b53f1c/0xd5401cbabc01f409 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x54f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efb23 expref: 946 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.728039:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x54f:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.728042:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x54f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.728047:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0251be lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x54f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.728052:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.728079:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000071b53f1c/0xd5401cbabc01f409 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x54f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb23 expref: 946 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.728092:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000071b53f1c/0xd5401cbabc01f409 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x54f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb23 expref: 946 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.728110:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943512427800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993064064:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.728142:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515476300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993063808:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.728146:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470cc0 going to sleep for 11 seconds 00000004:00080000:3.0:1783520935.728152:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351530ce00/2, rc 0, transno 4294975614 00000100:00100000:3.0:1783520935.728167:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993061632:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.728179:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.728221:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993063808 00000100:00100000:1.0:1783520935.728227:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993063808 00000100:00100000:3.0:1783520935.728230:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1783520935.728243:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993064192:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.728256:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44123 00000100:00100000:1.0:1783520935.728268:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363566ec50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+53:3626:x1870152993063808:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.728280:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1360 count 1 00000100:00100000:3.0:1783520935.728287:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.728290:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc0251cc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x550:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.728297:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.728305:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000061b76c4c/0xd5401cbabc01f3f4 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x550:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efb31 expref: 946 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.728311:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x550:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.728313:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x550:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.728316:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc0251cc lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x550:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.728331:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000061b76c4c/0xd5401cbabc01f3f4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x550:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb31 expref: 946 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.728338:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993064192 00010000:00010000:1.0:1783520935.728342:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000061b76c4c/0xd5401cbabc01f3f4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x550:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb31 expref: 946 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.728344:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993064192 00000100:00100000:1.0:1783520935.728383:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad2300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993064448:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.728423:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6300 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.728431:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.728443:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000009eb77304/0xd5401cbabc0251a2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x54a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.728449:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44124 00010000:00010000:1.0:1783520935.728458:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000009eb77304/0xd5401cbabc0251a2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x54a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.728465:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+54:3628:x1870152993064192:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.728467:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000009eb77304/0xd5401cbabc0251a2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x54a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.728476:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000009eb77304/0xd5401cbabc0251a2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x54a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520935.728481:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1361 count 1 00010000:00010000:1.0:1783520935.728490:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.728496:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0251d3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x551:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.728522:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002368256 00000100:00100000:0.0:1783520935.728527:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002368256 00010000:00010000:0.0:1783520935.728546:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000fd320b0d/0xd5401cbabc01f3df lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x54c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efaf9 expref: 947 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.728582:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002368128 00000100:00100000:1.0:1783520935.728584:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+55:3627:x1870152993062656:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2634us (2830us total) trans 4294975618 rc 0/0 00000100:00100000:0.0:1783520935.728588:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002368128 00000100:00100000:1.0:1783520935.728599:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44117 00010000:00010000:0.0:1783520935.728606:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000024b66d26/0xd5401cbabc01f3d8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x54b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efaeb expref: 948 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.728633:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351530ea00/2, rc 0, transno 4294975618 00000100:00100000:0.0:1783520935.728642:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35711 00000100:00100000:1.0:1783520935.728647:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993062656:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.728655:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943512425180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+948:2389:x1870153002368256:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.728667:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.728677:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000fd320b0d/0xd5401cbabc01f3df lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x54c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efaf9 expref: 948 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.728690:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:0.0:1783520935.728695:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000fd320b0d/0xd5401cbabc01f3df lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x54c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efaf9 expref: 948 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.728701:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993064576:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.728712:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f50faf44/0xd5401cbabc01f3fb lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x551:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efb3f expref: 948 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.728722:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x54c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.728731:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x54c:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.728735:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.728736:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x551:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.728741:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0251b0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x54c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.728743:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x551:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.728754:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0251d3 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x551:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.728755:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0251b0 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x54c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.728760:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993064576 00000100:00100000:1.0:1783520935.728765:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993064576 00000100:00100000:1.0:1783520935.728784:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44125 00010000:00010000:0.0:1783520935.728786:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.728793:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+55:3627:x1870152993064576:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.728802:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f50faf44/0xd5401cbabc01f3fb lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x551:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb3f expref: 947 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.728811:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1362 count 1 00000100:00100000:0.0:1783520935.728818:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943512425180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+947:2389:x1870153002368256:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 164us (524us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.728828:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0251da lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x552:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.728829:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f50faf44/0xd5401cbabc01f3fb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x551:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb3f expref: 947 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.728838:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35711 00010000:00010000:1.0:1783520935.728860:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b141858d/0xd5401cbabc01f402 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x552:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efb4d expref: 946 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.728864:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9e300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993064704:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.728874:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x552:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.728879:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x552:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.728880:0:9151:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000fd320b0d/0xd5401cbabc01f3df lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x54c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efaf9 expref: 946 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.728883:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0251da lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x552:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.728902:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515475500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993063424:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.728907:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872f00 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.728912:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b141858d/0xd5401cbabc01f402 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x552:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb4d expref: 945 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.728916:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0251b0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x54c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.728923:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b141858d/0xd5401cbabc01f402 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x552:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb4d expref: 945 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.728935:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0251b0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x54c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.728941:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993064832:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.728944:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0251b0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x54c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.728951:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0251b0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x54c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.728966:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6f00 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.728978:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0251b0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x54c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.729002:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.729010:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.729022:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0251b0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x54c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.729036:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfa000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.729040:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0251b0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x54c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.729049:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0251b0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x54c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.729063:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0251b0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x54c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.729071:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0251b0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x54c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.729086:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.729089:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0251a9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x54b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.729094:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.729097:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0251a9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x54b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.729102:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0251a9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x54b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.729110:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+57:3626:x1870152993063168:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2072us (2142us total) trans 4294975619 rc 0/0 00010000:00010000:1.0:1783520935.729113:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0251a9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x54b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.729123:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ad3800/2, rc 0, transno 4294975619 00000100:00100000:0.0:1783520935.729126:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44119 00010000:00010000:1.0:1783520935.729132:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0251a9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x54b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.729141:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993063168:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.729158:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.729162:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.729166:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520935.729175:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ad0000/2, rc 0, transno 4294975620 00000100:00100000:0.0:1783520935.729184:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993062784:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.729272:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993064960:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.729308:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.729342:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943512427800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993064064:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.729386:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993064960 00010000:00010000:3.0:1783520935.729387:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0251be lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x54f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.729392:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993064960 00000100:00100000:1.0:1783520935.729426:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993065088 00000100:00100000:1.0:1783520935.729431:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993065088 00000100:00100000:0.0:1783520935.729432:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44126 00000100:00100000:0.0:1783520935.729444:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+57:3625:x1870152993064960:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.729459:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1363 count 1 00000100:00100000:1.0:1783520935.729463:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44127 00010000:00010000:0.0:1783520935.729472:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0251e1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x553:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.729474:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517244000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+57:3626:x1870152993065088:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.729492:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006b01d0f7/0xd5401cbabc01f410 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x553:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efb5b expref: 944 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.729495:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1364 count 1 00010000:00010000:0.0:1783520935.729515:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x553:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.729519:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x553:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.729525:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc0251e8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x554:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.729526:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0251e1 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x553:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.729553:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005040146a/0xd5401cbabc01f417 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x554:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efb69 expref: 944 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.729559:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x554:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.729562:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x554:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.729567:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006b01d0f7/0xd5401cbabc01f410 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x553:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb5b expref: 944 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.729568:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc0251e8 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x554:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.729582:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002368640 00000100:00100000:3.0:1783520935.729589:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002368640 00010000:00010000:0.0:1783520935.729600:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006b01d0f7/0xd5401cbabc01f410 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x553:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb5b expref: 945 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.729611:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009d514afa/0xd5401cbabc01f3e6 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x54d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb07 expref: 945 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.729615:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515345500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993065216:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.729622:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005040146a/0xd5401cbabc01f417 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x554:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb69 expref: 945 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.729642:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005040146a/0xd5401cbabc01f417 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x554:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb69 expref: 945 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.729653:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002368512 00000100:00100000:1.0:1783520935.729657:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad0e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993065344:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.729659:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002368512 00000100:00100000:0.0:1783520935.729673:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470780 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.729678:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000071b53f1c/0xd5401cbabc01f409 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x54f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb23 expref: 946 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.729694:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002368384 00000100:00100000:1.0:1783520935.729695:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6600 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.729701:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002368384 00000100:00100000:3.0:1783520935.729707:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35713 00000100:00100000:3.0:1783520935.729719:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517246a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+947:2390:x1870153002368640:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.729723:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000003315ef/0xd5401cbabc01f3ed lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x54e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb15 expref: 947 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.729729:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.729735:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002368768 00010000:00010000:3.0:1783520935.729739:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009d514afa/0xd5401cbabc01f3e6 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x54d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb07 expref: 947 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.729741:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002368768 00000100:00100000:0.0:1783520935.729758:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35714 00010000:00010000:1.0:1783520935.729764:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000061b76c4c/0xd5401cbabc01f3f4 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x550:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb31 expref: 948 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.729771:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f3450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+948:2388:x1870153002368512:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.729783:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.729789:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000071b53f1c/0xd5401cbabc01f409 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x54f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb23 expref: 948 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.729798:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35715 00010000:00010000:0.0:1783520935.729803:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000071b53f1c/0xd5401cbabc01f409 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x54f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb23 expref: 948 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.729804:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009d514afa/0xd5401cbabc01f3e6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x54d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb07 expref: 948 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.729813:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517244a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+948:2388:x1870153002368384:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.729830:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.729839:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000003315ef/0xd5401cbabc01f3ed lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x54e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb15 expref: 948 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.729845:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000071b53f1c/0xd5401cbabc01f409 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x54f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb23 expref: 948 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.729864:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000003315ef/0xd5401cbabc01f3ed lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x54e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb15 expref: 948 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.729872:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x54f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.729883:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x54f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.729889:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x54e:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.729892:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0251be lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x54f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.729898:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x54e:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.729907:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0251b7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x54e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.729912:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0251be lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x54f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.729915:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0251b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x54e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.729945:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.729952:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.729953:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x54d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.729962:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x54d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.729967:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0251c5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x54d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.729971:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517244a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+947:2388:x1870153002368384:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 163us (350us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.729979:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f3450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+947:2388:x1870153002368512:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 210us (396us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.729989:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35715 00010000:00010000:3.0:1783520935.729991:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0251c5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x54d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.730001:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35714 00010000:00010000:3.0:1783520935.730014:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.730041:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517246a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+944:2390:x1870153002368640:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 324us (495us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.730050:0:35275:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000003315ef/0xd5401cbabc01f3ed lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x54e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb15 expref: 944 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.730052:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35713 00000100:00100000:0.0:1783520935.730066:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517246d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993063936:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.730088:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0251b7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x54e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.730098:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0251b7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x54e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.730105:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0251b7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x54e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.730115:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0251b7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x54e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.730128:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0251b7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x54e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.730144:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0251be lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x54f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.730145:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.730157:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0251be lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x54f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.730168:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0251be lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x54f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.730177:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0251be lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x54f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.730194:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:1.0:1783520935.730196:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993064832:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.730216:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0251da lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x552:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.730239:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002368896 00000100:00100000:0.0:1783520935.730243:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+58:3628:x1870152993063552:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2382us (2467us total) trans 4294975621 rc 0/0 00000100:00100000:1.0:1783520935.730245:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002368896 00000100:00100000:0.0:1783520935.730258:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44120 00010000:00010000:1.0:1783520935.730265:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f50faf44/0xd5401cbabc01f3fb lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x551:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb3f expref: 943 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.730290:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002369024 00000100:00100000:1.0:1783520935.730296:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35717 00000100:00100000:0.0:1783520935.730297:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002369024 00000100:00100000:1.0:1783520935.730309:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517245c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+944:2387:x1870153002368896:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.730318:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b141858d/0xd5401cbabc01f402 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x552:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb4d expref: 944 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.730321:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.730331:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f50faf44/0xd5401cbabc01f3fb lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x551:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb3f expref: 944 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.730350:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35718 00000100:00100000:0.0:1783520935.730378:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517246d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+944:2389:x1870153002369024:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.730385:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.730391:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b141858d/0xd5401cbabc01f402 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x552:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb4d expref: 944 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.730401:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f50faf44/0xd5401cbabc01f3fb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x551:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb3f expref: 944 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.730404:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b141858d/0xd5401cbabc01f402 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x552:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb4d expref: 944 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.730435:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b141858d/0xd5401cbabc01f402 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x552:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb4d expref: 944 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.730436:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x551:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.730447:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x551:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.730457:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0251d3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x551:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.730469:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x552:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.730478:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x552:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.730486:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0251d3 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x551:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.730486:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0251da lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x552:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.730518:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0251da lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x552:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.730525:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.730537:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.730552:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517245c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+943:2387:x1870153002368896:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 245us (393us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.730564:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35717 00000100:00100000:0.0:1783520935.730565:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517246d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+943:2389:x1870153002369024:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 201us (369us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.730575:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35718 00000100:00100000:3.0:1783520935.730593:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515475880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+58:3625:x1870152993063680:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2542us (2610us total) trans 4294975622 rc 0/0 00010000:00010000:1.0:1783520935.730593:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0251da lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x552:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.730612:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0251da lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x552:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.730613:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44121 00000100:00100000:0.0:1783520935.730623:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515345500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993065216:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.730631:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0251da lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x552:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.730642:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0251da lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x552:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.730644:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0251e1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x553:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.730650:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.730666:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000004:00080000:3.0:1783520935.730678:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515475180/2, rc 0, transno 4294975621 00000100:00100000:0.0:1783520935.730682:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002369408 00000100:00100000:0.0:1783520935.730687:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002369408 00000100:00100000:3.0:1783520935.730693:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515475180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993063552:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.730707:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005040146a/0xd5401cbabc01f417 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x554:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb69 expref: 942 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.730733:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002369152 00000100:00100000:0.0:1783520935.730740:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002369152 00000100:00100000:3.0:1783520935.730751:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:0.0:1783520935.730759:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006b01d0f7/0xd5401cbabc01f410 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x553:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb5b expref: 943 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.730761:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+58:3627:x1870152993064576:12345-0@lo:6:osp-syn-1-0.0 Request processed in 1966us (2026us total) trans 4294975623 rc 0/0 00000100:00100000:3.0:1783520935.730763:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993065472:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.730784:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44125 00000100:00100000:3.0:1783520935.730794:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.730810:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35719 00000100:00100000:0.0:1783520935.730822:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517245f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+943:2390:x1870153002369408:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.730831:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.730840:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005040146a/0xd5401cbabc01f417 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x554:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb69 expref: 943 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.730844:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993065472 00000100:00100000:3.0:1783520935.730849:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993065472 00010000:00010000:0.0:1783520935.730855:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005040146a/0xd5401cbabc01f417 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x554:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb69 expref: 943 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.730863:0:35280:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002369280 00000100:00100000:1.0:1783520935.730869:0:35280:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002369280 00010000:00010000:0.0:1783520935.730879:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x554:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.730883:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44128 00010000:00010000:0.0:1783520935.730886:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x554:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.730895:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc0251e8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x554:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.730897:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+58:3628:x1870152993065472:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.730914:0:35280:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98399 00002000:00080000:3.0:1783520935.730916:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1365 count 1 00010000:00010000:0.0:1783520935.730922:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc0251e8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x554:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.730931:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0251ef lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x555:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.730931:0:35280:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517245180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_011:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+20:251790:x1870153002369280:12345-192.168.204.33@tcp:101:lfs.0 00010000:00010000:0.0:1783520935.730954:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.730956:0:35280:0:(ldlm_lockd.c:1265:ldlm_handle_enqueue()) ### server-side enqueue handler START 00010000:00010000:3.0:1783520935.730962:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c34688b7/0xd5401cbabc01f41e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x555:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efb77 expref: 943 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.730975:0:35280:0:(ldlm_lockd.c:1343:ldlm_handle_enqueue()) ### server-side enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6c00/0xd5401cbabc0251f6 lrc: 2/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT gid 0 flags: 0x40000000000000 nid: local remote: 0xc275876b859f34ab expref: -99 pid: 35280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.730979:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517245f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+943:2390:x1870153002369408:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 157us (355us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.730986:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x555:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.730997:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x555:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.731000:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35719 00010000:00010000:3.0:1783520935.731004:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0251ef lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x555:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.731034:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c34688b7/0xd5401cbabc01f41e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x555:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb77 expref: 941 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.731039:0:35282:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009d514afa/0xd5401cbabc01f3e6 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x54d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb07 expref: 941 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.731046:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c34688b7/0xd5401cbabc01f41e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x555:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb77 expref: 941 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.731072:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517247480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993064320:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.731075:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a5f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993065600:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.731082:0:35280:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7e00/0xd5401cbabc0251fd lrc: 3/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x40000000000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.731090:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0251c5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x54d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.731096:0:35280:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7e00/0xd5401cbabc0251fd lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT flags: 0x50210000000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.731099:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0251c5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x54d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.731109:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c8729c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.731116:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0251c5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x54d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.731118:0:35280:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7e00/0xd5401cbabc0251fd lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT flags: 0x40210000000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.731125:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0251c5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x54d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.731133:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0251c5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x54d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.731147:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000004:00200000:1.0:1783520935.731168:0:35280:0:(mdt_handler.c:909:mdt_pack_attr2body()) [0x200000007:0x1:0x0]: returning size 69632 00000004:00200000:1.0:1783520935.731175:0:35280:0:(mdt_handler.c:1574:mdt_getattr_internal()) dirent count 0 stripe count 1 MDT count 2 00000004:00010000:1.0:1783520935.731197:0:35280:0:(mdt_handler.c:2507:mdt_getattr_name_lock()) ### Returning lock to client ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7e00/0xd5401cbabc0251fd lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT flags: 0x40210000000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.731228:0:35280:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6c00/0xd5401cbabc0251f6 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x44000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34ab expref: 22 pid: 35280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.731238:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+59:3628:x1870152993063296:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3111us (3186us total) trans 4294975624 rc 0/0 00000100:00100000:0.0:1783520935.731256:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44122 00010000:00010000:1.0:1783520935.731265:0:35280:0:(ldlm_lockd.c:1513:ldlm_handle_enqueue()) ### server-side enqueue handler, sending reply (err=0, rc=0) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7e00/0xd5401cbabc0251fd lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x40200000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34ab expref: 21 pid: 35280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.731274:0:35280:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x0 (ffff943504b72c00) 00010000:00010000:1.0:1783520935.731278:0:35280:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 13 00010000:00010000:1.0:1783520935.731281:0:35280:0:(ldlm_lockd.c:1592:ldlm_handle_enqueue()) ### server-side enqueue handler END (lock ffff943514fa7e00, rc 0) 00000004:00080000:0.0:1783520935.731289:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ad1500/2, rc 0, transno 4294975624 00000100:00100000:0.0:1783520935.731300:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993063296:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.731316:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0251e1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x553:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.731317:0:35280:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517245180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_011:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+21:251790:x1870153002369280:12345-192.168.204.33@tcp:101:lfs.0 Request processed in 390us (655us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.731335:0:35280:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98399 00010000:00010000:0.0:1783520935.731345:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0251e1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x553:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.731354:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0251e1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x553:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.731384:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0251e1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x553:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.731402:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000004:00080000:1.0:1783520935.731418:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ad3100/2, rc 0, transno 4294975623 00000100:00100000:1.0:1783520935.731434:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993064576:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.731448:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1783520935.731459:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993065728:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.731484:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+59:3625:x1870152993064960:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2040us (2174us total) trans 4294975625 rc 0/0 00000100:00100000:0.0:1783520935.731493:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44126 00000100:00100000:1.0:1783520935.731513:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.731537:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993065728 00000100:00100000:0.0:1783520935.731544:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993065728 00010000:00010000:1.0:1783520935.731566:0:8410:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f50faf44/0xd5401cbabc01f3fb lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x551:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb3f expref: 939 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.731582:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44129 00000100:00100000:3.0:1783520935.731587:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993065856 00000100:00100000:1.0:1783520935.731587:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9e300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993064704:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.731593:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993065856 00000100:00100000:0.0:1783520935.731595:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+59:3627:x1870152993065728:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.731609:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0251d3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x551:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.731611:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1366 count 1 00010000:00010000:1.0:1783520935.731624:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0251d3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x551:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.731628:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44130 00010000:00010000:0.0:1783520935.731628:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025204 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x556:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.731643:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517247b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+60:3626:x1870152993065856:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.731659:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0251d3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x551:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.731663:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007c0e94a8/0xd5401cbabc01f472 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x556:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efb85 expref: 938 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.731675:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0251d3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x551:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.731677:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x556:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.731688:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x556:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.731690:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0251d3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x551:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.731694:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025204 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x556:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.731714:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00002000:00080000:3.0:1783520935.731718:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1367 count 1 00010000:00010000:0.0:1783520935.731724:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007c0e94a8/0xd5401cbabc01f472 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x556:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb85 expref: 938 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.731732:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc02520b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x557:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.731737:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007c0e94a8/0xd5401cbabc01f472 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x556:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb85 expref: 938 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.731765:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000298ac3d1/0xd5401cbabc01f479 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x557:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efb93 expref: 938 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.731766:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515345500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993065984:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.731782:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x557:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.731789:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x557:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.731793:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc02520b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x557:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.731795:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+61:3628:x1870152993064192:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3335us (3508us total) trans 4294975626 rc 0/0 00000100:00100000:1.0:1783520935.731815:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44124 00000100:00100000:0.0:1783520935.731821:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470d80 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.731823:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000298ac3d1/0xd5401cbabc01f479 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x557:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb93 expref: 938 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.731833:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000298ac3d1/0xd5401cbabc01f479 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x557:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb93 expref: 938 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.731846:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a7b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993066112:192.168.204.33@tcp:104: 00000004:00080000:0.0:1783520935.731856:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351530f800/2, rc 0, transno 4294975626 00010000:00010000:1.0:1783520935.731865:0:35274:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005040146a/0xd5401cbabc01f417 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x554:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb69 expref: 938 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.731870:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872f00 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.731872:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993064192:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.731888:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad0e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993065344:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.731911:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc0251e8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x554:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.731924:0:20415:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000061b76c4c/0xd5401cbabc01f3f4 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x550:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb31 expref: 937 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.731927:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc0251e8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x554:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.731947:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc0251e8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x554:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.731952:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad2300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993064448:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.731966:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc0251e8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x554:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.731976:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc0251e8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x554:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.731977:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc0251cc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x550:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.731997:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc0251cc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x550:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.732001:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.732009:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc0251cc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x550:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.732024:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc0251cc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x550:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.732040:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc0251cc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x550:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.732072:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520935.732077:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993066240 00000100:00100000:3.0:1783520935.732083:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993066240 00000100:00100000:1.0:1783520935.732091:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517244000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+62:3626:x1870152993065088:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2618us (2748us total) trans 4294975627 rc 0/0 00000100:00100000:1.0:1783520935.732106:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44127 00000100:00100000:3.0:1783520935.732117:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44131 00000100:00100000:3.0:1783520935.732129:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517247800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+62:3626:x1870152993066240:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520935.732147:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1368 count 1 00000004:00080000:1.0:1783520935.732149:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435121a5180/2, rc 0, transno 4294975627 00000100:00100000:0.0:1783520935.732160:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363566ec50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+62:3626:x1870152993063808:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3893us (3981us total) trans 4294975628 rc 0/0 00010000:00010000:3.0:1783520935.732163:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025212 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x558:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.732164:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993065088:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.732181:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44123 00000004:00080000:1.0:1783520935.732185:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515476300/2, rc 0, transno 4294975628 00000100:00100000:1.0:1783520935.732193:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515476300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993063808:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.732244:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000892ca38a/0xd5401cbabc01f480 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x558:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efba1 expref: 936 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.732255:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x558:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.732260:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x558:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.732265:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025212 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x558:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.732272:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993066368:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.732310:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.732314:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993066496:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.732352:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:1.0:1783520935.732394:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435121a7480/2, rc 0, transno 4294975625 00000100:00100000:1.0:1783520935.732406:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993064960:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.732411:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002369536 00000100:00100000:0.0:1783520935.732415:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002369536 00010000:00010000:0.0:1783520935.732432:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c34688b7/0xd5401cbabc01f41e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x555:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb77 expref: 937 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.732444:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1783520935.732453:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35721 00000100:00100000:1.0:1783520935.732455:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993066624:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.732462:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f1850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+937:2388:x1870153002369536:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.732470:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.732474:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c34688b7/0xd5401cbabc01f41e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x555:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb77 expref: 937 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.732482:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c34688b7/0xd5401cbabc01f41e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x555:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb77 expref: 937 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.732490:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.732516:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c34688b7/0xd5401cbabc01f41e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x555:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb77 expref: 937 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.732520:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000892ca38a/0xd5401cbabc01f480 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x558:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efba1 expref: 937 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.732536:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x555:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.732539:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993066368 00000100:00100000:1.0:1783520935.732546:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993066368 00010000:00010000:0.0:1783520935.732546:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x555:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.732556:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0251ef lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x555:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.732565:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0251ef lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x555:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.732569:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993066496 00000100:00100000:1.0:1783520935.732574:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993066496 00000100:00100000:1.0:1783520935.732593:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993066624 00010000:00010000:3.0:1783520935.732597:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000892ca38a/0xd5401cbabc01f480 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x558:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efba1 expref: 936 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.732597:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993066624 00010000:00010000:0.0:1783520935.732598:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.732617:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993066752:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.732630:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f1850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+936:2388:x1870153002369536:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 168us (309us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.732636:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44133 00000100:00100000:0.0:1783520935.732648:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35721 00000100:00100000:1.0:1783520935.732653:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515346d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+64:3628:x1870152993066496:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.732663:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872b40 going to sleep for 11 seconds 00002000:00080000:1.0:1783520935.732672:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1370 count 1 00010000:00010000:1.0:1783520935.732688:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025220 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x55a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.732702:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44134 00010000:00010000:1.0:1783520935.732706:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d3bb955b/0xd5401cbabc01f48e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x55a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efbbd expref: 935 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.732718:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+64:3626:x1870152993066624:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.732720:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x55a:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.732728:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x55a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.732734:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025220 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x55a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.732735:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1371 count 1 00010000:00010000:0.0:1783520935.732749:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025227 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x55b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.732767:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.732768:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002c826d9d/0xd5401cbabc01f495 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x55b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efbcb expref: 935 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.732775:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d3bb955b/0xd5401cbabc01f48e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x55a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbbd expref: 935 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.732783:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x55b:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.732794:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x55b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.732800:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025227 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x55b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.732803:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d3bb955b/0xd5401cbabc01f48e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x55a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbbd expref: 935 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.732831:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad2680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993067008:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.732843:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002c826d9d/0xd5401cbabc01f495 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x55b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbcb expref: 935 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.732862:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6540 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.732869:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002c826d9d/0xd5401cbabc01f495 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x55b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbcb expref: 935 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.732887:0:35277:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002369792 00000100:00100000:0.0:1783520935.732892:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515344700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993067136:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.732893:0:35277:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002369792 00000100:00100000:3.0:1783520935.732911:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a7b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993066112:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.732929:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470cc0 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.732930:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc02520b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x557:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.732936:0:35277:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98400 00000100:00100000:1.0:1783520935.732950:0:35277:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_008:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+21:251790:x1870153002369792:12345-192.168.204.33@tcp:101:lfs.0 00010000:00010000:1.0:1783520935.732960:0:35277:0:(ldlm_lockd.c:1265:ldlm_handle_enqueue()) ### server-side enqueue handler START 00010000:00010000:1.0:1783520935.732973:0:35277:0:(ldlm_lockd.c:1343:ldlm_handle_enqueue()) ### server-side enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7800/0xd5401cbabc02522e lrc: 2/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT gid 0 flags: 0x40000000000000 nid: local remote: 0xc275876b859f34b2 expref: -99 pid: 35277 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.733009:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435121a6680/2, rc 0, transno 4294975629 00000100:00100000:0.0:1783520935.733025:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993065472:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.733049:0:35277:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6a00/0xd5401cbabc025235 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x40000000000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.733071:0:35277:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6a00/0xd5401cbabc025235 lrc: 3/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210000000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.733083:0:35277:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6a00/0xd5401cbabc025235 lrc: 3/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.733111:0:35277:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc02523c lrc: 3/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x40000000000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.733123:0:35277:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc02523c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.733132:0:35277:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc02523c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210000000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.733182:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.733209:0:35277:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6a00/0xd5401cbabc025235 lrc: 4/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.733217:0:35277:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6a00/0xd5401cbabc025235 lrc: 4/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.733223:0:35277:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6a00/0xd5401cbabc025235 lrc: 3/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.733233:0:35277:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6a00/0xd5401cbabc025235 lrc: 4/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.733239:0:35277:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6a00 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:1.0:1783520935.733243:0:35277:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6a00/0xd5401cbabc025235 lrc: 4/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.733244:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002369920 00000100:00100000:3.0:1783520935.733251:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002369920 00010000:00010000:1.0:1783520935.733254:0:35277:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6a00/0xd5401cbabc025235 lrc: 5/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40218400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.733265:0:35277:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x0 (ffff943504b72c00) 00010000:00010000:1.0:1783520935.733268:0:35277:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:3.0:1783520935.733271:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000298ac3d1/0xd5401cbabc01f479 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x557:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efb93 expref: 936 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.733272:0:35277:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6a00/0xd5401cbabc025235 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x44a19400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.733291:0:35277:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6a00/0xd5401cbabc025235 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x44a19400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.733304:0:35277:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc02523c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.733311:0:35277:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc02523c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.733312:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35722 00010000:00010000:1.0:1783520935.733322:0:35277:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc02523c lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.733329:0:35277:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6800 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:1.0:1783520935.733332:0:35277:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc02523c lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.733342:0:35277:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc02523c lrc: 4/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40218400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.733349:0:35277:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x5ba6781 (ffff943507a8d400) 00010000:00010000:1.0:1783520935.733352:0:35277:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:1.0:1783520935.733355:0:35277:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc02523c lrc: 2/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.733383:0:35277:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc02523c lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.733421:0:35277:0:(ldlm_lockd.c:1513:ldlm_handle_enqueue()) ### server-side enqueue handler, sending reply (err=301, rc=0) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7800/0xd5401cbabc02522e lrc: 1/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x44000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34b2 expref: 22 pid: 35277 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.733430:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515345f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+936:2387:x1870153002369920:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.733434:0:35277:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x0 (ffff943504b72c00) 00010000:00010000:1.0:1783520935.733444:0:35277:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 1 00000004:00080000:0.0:1783520935.733447:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515475c00/2, rc 0, transno 4294975622 00010000:00010000:1.0:1783520935.733452:0:35277:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7800/0xd5401cbabc02522e lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x44801000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34b2 expref: 22 pid: 35277 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.733464:0:35277:0:(ldlm_lockd.c:1592:ldlm_handle_enqueue()) ### server-side enqueue handler END (lock ffff943514fa7800, rc 0) 00000100:00100000:0.0:1783520935.733469:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515475c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993063680:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.733479:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1783520935.733487:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993067264:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.733515:0:35277:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_008:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+21:251790:x1870153002369792:12345-192.168.204.33@tcp:101:lfs.0 Request processed in 553us (738us total) trans 0 rc 301/301 00000100:00100000:1.0:1783520935.733526:0:35277:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98400 00000100:00100000:0.0:1783520935.733526:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1783520935.733537:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943512426300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993067392:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.733564:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.733593:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993067392 00000100:00100000:3.0:1783520935.733600:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993067392 00000100:00100000:0.0:1783520935.733602:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.733632:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993067264 00000100:00100000:3.0:1783520935.733637:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44135 00000100:00100000:1.0:1783520935.733639:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993067264 00000100:00100000:3.0:1783520935.733648:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515345880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+66:3628:x1870152993067392:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.733660:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025204 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x556:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520935.733663:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1373 count 1 00000100:00100000:1.0:1783520935.733676:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44136 00010000:00010000:3.0:1783520935.733680:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025243 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x55d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.733692:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515346680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+66:3625:x1870152993067264:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.733693:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025204 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x556:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.733709:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025204 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x556:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.733712:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004aa12b46/0xd5401cbabc01f4a3 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x55d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efbe7 expref: 935 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.733716:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1372 count 1 00010000:00010000:0.0:1783520935.733720:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025204 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x556:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.733734:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc02524a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x55c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.733737:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x55d:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.733746:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x55d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.733748:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.733759:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025243 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x55d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.733766:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000861f105f/0xd5401cbabc01f49c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x55c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efbd9 expref: 935 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.733790:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x55c:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.733799:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x55c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.733800:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004aa12b46/0xd5401cbabc01f4a3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x55d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbe7 expref: 935 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.733805:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc02524a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x55c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.733823:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004aa12b46/0xd5401cbabc01f4a3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x55d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbe7 expref: 935 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.733826:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+67:3627:x1870152993065728:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2234us (2330us total) trans 4294975630 rc 0/0 00000100:00100000:0.0:1783520935.733845:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44129 00010000:00010000:1.0:1783520935.733849:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000861f105f/0xd5401cbabc01f49c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x55c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbd9 expref: 934 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.733851:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943512424a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993067520:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.733871:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000861f105f/0xd5401cbabc01f49c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x55c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbd9 expref: 934 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.733893:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993067648:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.733894:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872f00 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.733919:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc02520b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x557:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.733928:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6840 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.733930:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc02520b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x557:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.733949:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc02520b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x557:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.733956:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435121a5500/2, rc 0, transno 4294975630 00010000:00010000:0.0:1783520935.733963:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc02520b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x557:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.733973:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993065728:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.734004:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.734018:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1783520935.734031:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943512425180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993067776:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.734064:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.734095:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517247b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+67:3626:x1870152993065856:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2453us (2519us total) trans 4294975631 rc 0/0 00010000:00010000:1.0:1783520935.734104:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.734109:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44130 00000100:00100000:3.0:1783520935.734119:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993067776 00000100:00100000:3.0:1783520935.734125:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993067776 00000004:00080000:1.0:1783520935.734133:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435121a4000/2, rc 0, transno 4294975631 00000100:00100000:1.0:1783520935.734147:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993065856:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.734195:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1783520935.734204:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993067904:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.734236:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44137 00000100:00100000:1.0:1783520935.734238:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.734251:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+67:3627:x1870152993067776:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.734267:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1374 count 1 00000100:00100000:3.0:1783520935.734277:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993067904 00010000:00010000:0.0:1783520935.734281:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025251 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x55e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.734282:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993067904 00000100:00100000:1.0:1783520935.734292:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad2680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993067008:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.734299:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d6609c39/0xd5401cbabc01f4aa lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x55e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efbf5 expref: 933 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.734305:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x55e:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.734308:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x55e:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.734311:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025251 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x55e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.734313:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025220 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x55a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.734317:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44138 00000100:00100000:3.0:1783520935.734331:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+68:3626:x1870152993067904:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520935.734347:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1375 count 1 00010000:00010000:0.0:1783520935.734348:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d6609c39/0xd5401cbabc01f4aa lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x55e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbf5 expref: 933 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.734349:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002370304 00000100:00100000:1.0:1783520935.734379:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002370304 00010000:00010000:3.0:1783520935.734380:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025258 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x55f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.734399:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d3bb955b/0xd5401cbabc01f48e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x55a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbbd expref: 934 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.734406:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d6609c39/0xd5401cbabc01f4aa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x55e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbf5 expref: 934 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.734415:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000fbf12d47/0xd5401cbabc01f4b1 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x55f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efc03 expref: 934 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.734436:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002370560 00010000:00010000:3.0:1783520935.734438:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x55f:0x0].0x0 since=37s v=(0 0) 00000100:00100000:1.0:1783520935.734442:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002370560 00010000:00010000:3.0:1783520935.734447:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x55f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.734453:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025258 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x55f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.734453:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515346a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993068032:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.734460:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002c826d9d/0xd5401cbabc01f495 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x55b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbcb expref: 935 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.734493:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000fbf12d47/0xd5401cbabc01f4b1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x55f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efc03 expref: 935 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.734510:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470240 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.734516:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002370176 00000100:00100000:1.0:1783520935.734522:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002370176 00000100:00100000:0.0:1783520935.734537:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35724 00010000:00010000:1.0:1783520935.734543:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000892ca38a/0xd5401cbabc01f480 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x558:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efba1 expref: 937 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.734552:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517244a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+937:2387:x1870153002370304:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.734568:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.734577:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d3bb955b/0xd5401cbabc01f48e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x55a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbbd expref: 937 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.734593:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d3bb955b/0xd5401cbabc01f48e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x55a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbbd expref: 937 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.734596:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000fbf12d47/0xd5401cbabc01f4b1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x55f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efc03 expref: 935 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.734598:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35726 00000100:00100000:1.0:1783520935.734616:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517246d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+937:2387:x1870153002370432:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.734629:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d3bb955b/0xd5401cbabc01f48e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x55a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbbd expref: 937 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.734631:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943512427800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993068160:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.734631:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.734642:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a8dfb240/0xd5401cbabc01f487 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x559:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbaf expref: 937 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.734663:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x55a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.734666:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a8dfb240/0xd5401cbabc01f487 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x559:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbaf expref: 936 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.734668:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c8729c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.734672:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x55a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.734682:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025220 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x55a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.734693:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x559:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.734694:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025220 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x55a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.734703:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x559:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.734705:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35727 00010000:00010000:1.0:1783520935.734715:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc025219 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x559:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.734722:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517245c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+936:2387:x1870153002370176:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.734726:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.734736:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc025219 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x559:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.734739:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.734750:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000892ca38a/0xd5401cbabc01f480 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x558:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efba1 expref: 936 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.734754:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517244a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+936:2387:x1870153002370304:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 204us (438us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.734768:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.734774:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35724 00010000:00010000:3.0:1783520935.734779:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000892ca38a/0xd5401cbabc01f480 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x558:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efba1 expref: 936 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.734793:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x558:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.734793:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517246d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+936:2387:x1870153002370432:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 182us (315us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.734798:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x558:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.734805:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025212 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x558:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.734809:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35726 00010000:00010000:3.0:1783520935.734824:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025212 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x558:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.734843:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025220 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x55a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.734849:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.734854:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025220 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x55a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.734863:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025220 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x55a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.734868:0:35282:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002c826d9d/0xd5401cbabc01f495 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x55b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbcb expref: 933 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.734877:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517245c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+933:2387:x1870153002370176:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 155us (459us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.734897:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35727 00010000:00010000:1.0:1783520935.734898:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025220 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x55a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.734901:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515344700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993067136:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.734925:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.734927:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025227 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x55b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.734937:0:8408:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000892ca38a/0xd5401cbabc01f480 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x558:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efba1 expref: 930 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.734948:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025227 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x55b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.734962:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993066752:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.734969:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025227 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x55b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.734977:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025227 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x55b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.734980:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025212 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x558:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.735000:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025227 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x55b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.735006:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025212 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x558:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.735008:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515346d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+69:3628:x1870152993066496:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2357us (2655us total) trans 4294975632 rc 0/0 00000100:00100000:1.0:1783520935.735028:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44133 00010000:00010000:0.0:1783520935.735031:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520935.735033:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025212 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x558:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.735055:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025212 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x558:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.735065:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:3.0:1783520935.735066:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025212 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x558:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.735073:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025220 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x55a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.735082:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.735085:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to regular blp list, 3 blwis in pool 00010000:00010000:1.0:1783520935.735086:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025220 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x55a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.735097:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025220 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x55a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.735099:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+70:3626:x1870152993066624:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2386us (2612us total) trans 4294975633 rc 0/0 00000100:00100000:0.0:1783520935.735114:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44134 00010000:00010000:1.0:1783520935.735119:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025220 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x55a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.735128:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025220 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x55a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.735147:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ad0380/2, rc 0, transno 4294975632 00000100:00100000:0.0:1783520935.735160:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993066496:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.735161:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00000100:00100000:3.0:1783520935.735167:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517247800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+69:3626:x1870152993066240:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3041us (3218us total) trans 4294975635 rc 0/0 00010000:00010000:1.0:1783520935.735168:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025227 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x55b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.735180:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44131 00010000:00010000:1.0:1783520935.735191:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.735197:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025227 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x55b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.735222:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025227 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x55b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.735247:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025227 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x55b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.735259:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025227 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x55b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.735280:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.735283:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025212 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x558:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.735289:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.735294:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025212 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x558:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.735302:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025212 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x558:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.735315:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025212 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x558:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.735320:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025212 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x558:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.735329:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.735331:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.735332:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520935.735404:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ad0e00/2, rc 0, transno 4294975634 00000100:00100000:1.0:1783520935.735416:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993066368:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.735436:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993068288:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.735456:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.735478:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993068288 00000004:00080000:1.0:1783520935.735480:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ad2300/2, rc 0, transno 4294975633 00000100:00100000:3.0:1783520935.735483:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993068288 00000100:00100000:1.0:1783520935.735486:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993066624:0@lo:6:osp-syn-1-0.0 00000004:00080000:1.0:1783520935.735511:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517246680/2, rc 0, transno 4294975635 00000100:00100000:1.0:1783520935.735516:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517246680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993066240:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.735521:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517245880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993068416:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.735538:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44139 00000100:00100000:3.0:1783520935.735547:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+69:3626:x1870152993068288:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.735553:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00080000:3.0:1783520935.735559:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1376 count 1 00000100:00100000:1.0:1783520935.735561:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1783520935.735569:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517246a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993068672:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.735570:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02525f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x560:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.735582:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:3.0:1783520935.735583:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d0ac06e5/0xd5401cbabc01f4b8 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x560:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efc11 expref: 929 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.735593:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x560:0x0].0x0 since=37s v=(0 0) 00000100:00100000:1.0:1783520935.735596:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.735596:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517245f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993068544:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.735601:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x560:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.735608:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02525f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x560:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.735628:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993068672 00000100:00100000:1.0:1783520935.735632:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993068672 00000100:00100000:0.0:1783520935.735636:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.735639:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d0ac06e5/0xd5401cbabc01f4b8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x560:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efc11 expref: 929 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.735651:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d0ac06e5/0xd5401cbabc01f4b8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x560:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efc11 expref: 929 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.735664:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993068800:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.735666:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993068544 00000100:00100000:1.0:1783520935.735669:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993068544 00000100:00100000:0.0:1783520935.735691:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44141 00000100:00100000:1.0:1783520935.735699:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44142 00000100:00100000:0.0:1783520935.735704:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515344700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+72:3627:x1870152993068416:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.735711:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515345c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+72:3627:x1870152993068544:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.735712:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607eb8a80 going to sleep for 11 seconds 00002000:00080000:0.0:1783520935.735719:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1377 count 1 00002000:00080000:1.0:1783520935.735728:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1378 count 1 00010000:00010000:0.0:1783520935.735731:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02526d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x561:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.735739:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025274 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x562:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.735765:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000044d843db/0xd5401cbabc01f505 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x562:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efcb9 expref: 929 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.735766:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ab6fd44b/0xd5401cbabc01f4bf lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x561:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efc1f expref: 929 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.735788:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x562:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.735793:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x561:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.735794:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x562:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.735805:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025274 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x562:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.735805:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x561:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.735816:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02526d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x561:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.735836:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000044d843db/0xd5401cbabc01f505 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x562:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efcb9 expref: 929 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.735844:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000044d843db/0xd5401cbabc01f505 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x562:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efcb9 expref: 929 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.735854:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993068928:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.735854:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ab6fd44b/0xd5401cbabc01f4bf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x561:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efc1f expref: 929 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.735878:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6540 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.735882:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ab6fd44b/0xd5401cbabc01f4bf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x561:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efc1f expref: 929 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.735917:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515344000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993069184:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.735949:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470180 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.736955:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943512424a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993067520:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.736956:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943512427800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993068160:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.736975:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025243 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x55d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.736976:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025258 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x55f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.736987:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993067648:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.737011:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc02524a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x55c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.737011:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515346a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993068032:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.737019:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002370816 00000100:00100000:3.0:1783520935.737025:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002370816 00010000:00010000:0.0:1783520935.737032:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025251 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x55e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.737045:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000fbf12d47/0xd5401cbabc01f4b1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x55f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efc03 expref: 930 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.737066:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002370944 00000100:00100000:3.0:1783520935.737072:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35728 00000100:00100000:1.0:1783520935.737072:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002370944 00000100:00100000:0.0:1783520935.737086:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002370688 00000100:00100000:3.0:1783520935.737087:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f2c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+931:2388:x1870153002370816:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.737091:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002370688 00010000:00010000:1.0:1783520935.737093:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000861f105f/0xd5401cbabc01f49c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x55c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbd9 expref: 931 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.737098:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.737107:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000fbf12d47/0xd5401cbabc01f4b1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x55f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efc03 expref: 932 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.737108:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004aa12b46/0xd5401cbabc01f4a3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x55d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbe7 expref: 932 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.737136:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002371072 00000100:00100000:1.0:1783520935.737141:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002371072 00000100:00100000:0.0:1783520935.737145:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35729 00000100:00100000:0.0:1783520935.737156:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517245500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+933:2390:x1870153002370944:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.737162:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d6609c39/0xd5401cbabc01f4aa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x55e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbf5 expref: 933 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.737170:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.737178:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000861f105f/0xd5401cbabc01f49c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x55c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbd9 expref: 933 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.737183:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000fbf12d47/0xd5401cbabc01f4b1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x55f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efc03 expref: 933 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.737184:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35730 00000100:00100000:1.0:1783520935.737196:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517247480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+933:2388:x1870153002370688:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.737198:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000861f105f/0xd5401cbabc01f49c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x55c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbd9 expref: 933 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.737203:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000fbf12d47/0xd5401cbabc01f4b1 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x55f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efc03 expref: 933 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.737204:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.737214:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004aa12b46/0xd5401cbabc01f4a3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x55d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbe7 expref: 933 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.737215:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000861f105f/0xd5401cbabc01f49c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x55c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbd9 expref: 933 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.737220:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x55f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.737227:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x55f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.737242:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x55c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.737244:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004aa12b46/0xd5401cbabc01f4a3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x55d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbe7 expref: 931 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.737250:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x55c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.737261:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc02524a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x55c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.737270:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004aa12b46/0xd5401cbabc01f4a3 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x55d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbe7 expref: 931 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.737274:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc02524a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x55c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.737285:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025258 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x55f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.737290:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x55d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.737294:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025258 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x55f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.737295:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x55d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.737301:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025243 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x55d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.737301:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.737315:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025243 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x55d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.737339:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.737341:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517245500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+930:2390:x1870153002370944:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 180us (304us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.737353:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35729 00010000:00010000:3.0:1783520935.737390:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.737414:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002371456 00000100:00100000:3.0:1783520935.737418:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f2c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+929:2388:x1870153002370816:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 334us (428us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.737418:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002371456 00000100:00100000:1.0:1783520935.737423:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517247480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+929:2388:x1870153002370688:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 227us (348us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.737430:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35728 00010000:00010000:0.0:1783520935.737435:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d0ac06e5/0xd5401cbabc01f4b8 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x560:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efc11 expref: 930 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.737439:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35730 00000100:00100000:0.0:1783520935.737458:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002371584 00000100:00100000:0.0:1783520935.737462:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002371584 00000100:00100000:1.0:1783520935.737463:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002371328 00000100:00100000:1.0:1783520935.737468:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002371328 00010000:00010000:0.0:1783520935.737477:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ab6fd44b/0xd5401cbabc01f4bf lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x561:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efc1f expref: 930 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.737482:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000044d843db/0xd5401cbabc01f505 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x562:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efcb9 expref: 930 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.737523:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35731 00000100:00100000:1.0:1783520935.737532:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35732 00000100:00100000:0.0:1783520935.737534:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514f9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+931:2390:x1870153002371072:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.737545:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514f99c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+931:2387:x1870153002371456:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.737545:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.737554:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d6609c39/0xd5401cbabc01f4aa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x55e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbf5 expref: 931 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.737558:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520935.737565:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35734 00010000:00010000:1.0:1783520935.737570:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d0ac06e5/0xd5401cbabc01f4b8 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x560:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efc11 expref: 931 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.737573:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d6609c39/0xd5401cbabc01f4aa lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x55e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbf5 expref: 931 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.737580:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351510b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+931:2388:x1870153002371328:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.737594:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.737599:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d0ac06e5/0xd5401cbabc01f4b8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x560:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efc11 expref: 931 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.737603:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000044d843db/0xd5401cbabc01f505 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x562:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efcb9 expref: 931 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.737604:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d6609c39/0xd5401cbabc01f4aa lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x55e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efbf5 expref: 931 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.737631:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x55e:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.737633:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x560:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.737637:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x55e:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.737643:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x560:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.737648:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000044d843db/0xd5401cbabc01f505 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x562:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efcb9 expref: 930 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.737648:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025251 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x55e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.737652:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02525f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x560:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.737668:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025251 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x55e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.737672:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x562:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.737679:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x562:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.737680:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02525f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x560:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.737688:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025274 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x562:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.737692:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.737707:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025274 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x562:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.737714:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.737717:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514f9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+930:2390:x1870153002371072:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 186us (610us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.737725:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35731 00000100:00100000:1.0:1783520935.737745:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514f99c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+929:2387:x1870153002371456:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 201us (512us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.737746:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35735 00000100:00100000:0.0:1783520935.737761:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357f050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+929:2388:x1870153002371712:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.737769:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35732 00010000:00010000:0.0:1783520935.737777:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.737779:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.737784:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002b1bc113/0xd5401cbabc01f50c lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x563:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efcc7 expref: 927 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.737797:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002b1bc113/0xd5401cbabc01f50c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x563:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efcc7 expref: 927 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.737799:0:35278:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002371200 00000100:00100000:3.0:1783520935.737804:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351510b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+927:2388:x1870153002371328:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 228us (501us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.737804:0:35278:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002371200 00010000:00010000:0.0:1783520935.737813:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x563:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.737814:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35734 00010000:00010000:0.0:1783520935.737817:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x563:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.737821:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025266 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x563:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.737829:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025266 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x563:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.737843:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.737844:0:35278:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98401 00010000:00010000:3.0:1783520935.737856:0:9151:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d0ac06e5/0xd5401cbabc01f4b8 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x560:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efc11 expref: 926 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.737857:0:35278:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514f98e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_009:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+21:251790:x1870153002371200:12345-192.168.204.33@tcp:34:lfs.0 00000100:00100000:0.0:1783520935.737871:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357f050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+926:2388:x1870153002371712:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 113us (530us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.737875:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993068800:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.737885:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35735 00010000:00010000:1.0:1783520935.737901:0:35278:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6a00/0xd5401cbabc02527b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x40000000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.737910:0:35278:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6a00/0xd5401cbabc02527b lrc: 3/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x50210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.737918:0:35278:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6a00/0xd5401cbabc02527b lrc: 3/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.737928:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02525f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x560:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.737933:0:35278:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6c00/0xd5401cbabc025282 lrc: 3/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x40000000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.737939:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025258 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x55f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.737952:0:35278:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6c00/0xd5401cbabc025282 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.737956:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02525f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x560:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.737961:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025258 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x55f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.737971:0:35278:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6c00/0xd5401cbabc025282 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.737986:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025258 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x55f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.737996:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02525f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x560:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.738004:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025258 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x55f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.738011:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02525f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x560:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.738022:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02525f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x560:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.738031:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520935.738037:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to regular blp list, 3 blwis in pool 00010000:00010000:1.0:1783520935.738046:0:35278:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7200/0xd5401cbabc025289 lrc: 3/1,0 mode: --/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x40000000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.738055:0:35278:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7200/0xd5401cbabc025289 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x50210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.738060:0:35278:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7200/0xd5401cbabc025289 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00000004:00200000:1.0:1783520935.738092:0:35278:0:(mdt_handler.c:909:mdt_pack_attr2body()) [0x200000402:0x72a:0x0]: returning size 131072 00000004:00200000:1.0:1783520935.738095:0:35278:0:(mdt_handler.c:1574:mdt_getattr_internal()) dirent count 0 stripe count 1 MDT count 2 00000004:00010000:1.0:1783520935.738103:0:35278:0:(mdt_handler.c:2507:mdt_getattr_name_lock()) ### Returning lock to client ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7200/0xd5401cbabc025289 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.738111:0:35278:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6a00/0xd5401cbabc02527b lrc: 4/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.738116:0:35278:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6a00/0xd5401cbabc02527b lrc: 4/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x50210400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.738121:0:35278:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6a00/0xd5401cbabc02527b lrc: 3/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x50210400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.738125:0:35278:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6a00/0xd5401cbabc02527b lrc: 4/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40210400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.738127:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+75:3626:x1870152993068288:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2579us (2669us total) trans 4294975636 rc 0/0 00010000:00010000:1.0:1783520935.738132:0:35278:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6a00 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:1.0:1783520935.738136:0:35278:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6a00/0xd5401cbabc02527b lrc: 4/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40210400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.738142:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44139 00000100:00100000:0.0:1783520935.738142:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+75:3626:x1870152993067904:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3812us (3904us total) trans 4294975638 rc 0/0 00010000:00010000:1.0:1783520935.738150:0:35278:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6a00/0xd5401cbabc02527b lrc: 5/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40218400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.738167:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44138 00010000:00010000:1.0:1783520935.738168:0:35278:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x0 (ffff943504b72c00) 00010000:00010000:1.0:1783520935.738173:0:35278:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:1.0:1783520935.738177:0:35278:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6a00/0xd5401cbabc02527b lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x44a19400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.738184:0:35278:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6a00/0xd5401cbabc02527b lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x44a19400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.738192:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025251 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x55e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.738204:0:35278:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6c00/0xd5401cbabc025282 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.738215:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025251 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x55e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.738221:0:35278:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6c00/0xd5401cbabc025282 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.738242:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025251 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x55e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.738247:0:35278:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6c00/0xd5401cbabc025282 lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.738250:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025251 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x55e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.738268:0:35278:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6c00 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:1.0:1783520935.738275:0:35278:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6c00/0xd5401cbabc025282 lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.738282:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to regular blp list, 3 blwis in pool 00010000:00010000:1.0:1783520935.738285:0:35278:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6c00/0xd5401cbabc025282 lrc: 4/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40218400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.738302:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1783520935.738305:0:35278:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x5ba6781 (ffff943507a8d900) 00010000:00010000:3.0:1783520935.738309:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02525f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x560:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.738312:0:35278:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:1.0:1783520935.738318:0:35278:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6c00/0xd5401cbabc025282 lrc: 2/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.738333:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.738337:0:35278:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6c00/0xd5401cbabc025282 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.738341:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02525f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x560:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.738349:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+73:3627:x1870152993067776:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4101us (4286us total) trans 4294975639 rc 0/0 00010000:00010000:3.0:1783520935.738383:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02525f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x560:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.738385:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44137 00010000:00010000:1.0:1783520935.738403:0:35278:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7200/0xd5401cbabc025289 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x50210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.738409:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02525f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x560:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.738419:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943512425180/2, rc 0, transno 4294975639 00010000:00010000:1.0:1783520935.738423:0:35278:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7200/0xd5401cbabc025289 lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x50210400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.738434:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02525f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x560:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.738434:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943512425180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993067776:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.738437:0:35278:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7200/0xd5401cbabc025289 lrc: 3/0,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40210400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.738459:0:35278:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7200 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:1.0:1783520935.738467:0:35278:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7200/0xd5401cbabc025289 lrc: 3/0,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40210400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.738468:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.738472:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.738476:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.738484:0:35278:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7200/0xd5401cbabc025289 lrc: 4/0,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40218400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.738508:0:35278:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x72a:0x0].0x0 (ffff943515473e00) 00010000:00010000:1.0:1783520935.738514:0:35278:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:1.0:1783520935.738518:0:35278:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7200/0xd5401cbabc025289 lrc: 2/0,0 mode: --/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x44a19400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.738524:0:35282:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ab6fd44b/0xd5401cbabc01f4bf lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x561:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efc1f expref: 923 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.738527:0:35278:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7200/0xd5401cbabc025289 lrc: 0/0,0 mode: --/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x44a19400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.738553:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515344000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993069184:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.738569:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02526d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x561:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.738581:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02526d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x561:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.738589:0:35278:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514f98e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_009:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+21:251790:x1870153002371200:12345-192.168.204.33@tcp:34:lfs.0 Request processed in 734us (1452us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.738591:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02526d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x561:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.738607:0:35278:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98401 00010000:00010000:0.0:1783520935.738608:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02526d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x561:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.738619:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02526d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x561:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.738636:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.738641:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351510b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993069312:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.738677:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:1.0:1783520935.738713:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517246a00/2, rc 0, transno 4294975637 00000100:00100000:0.0:1783520935.738720:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515344700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+73:3627:x1870152993068416:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3019us (3166us total) trans 4294975640 rc 0/0 00000100:00100000:1.0:1783520935.738729:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517246a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993068672:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.738734:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44141 00000100:00100000:3.0:1783520935.738740:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993069312 00000100:00100000:3.0:1783520935.738744:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993069312 00000004:00080000:1.0:1783520935.738750:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ad2680/2, rc 0, transno 4294975636 00000100:00100000:1.0:1783520935.738758:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993068288:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.738766:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993069440:0@lo:6:osp-syn-1-0.0 00000004:00080000:1.0:1783520935.738773:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ad0a80/2, rc 0, transno 4294975638 00000100:00100000:3.0:1783520935.738776:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44143 00000100:00100000:3.0:1783520935.738788:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+73:3626:x1870152993069312:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.738789:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993067904:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.738798:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00080000:3.0:1783520935.738809:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1380 count 1 00010000:00010000:1.0:1783520935.738827:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000044d843db/0xd5401cbabc01f505 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x562:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efcb9 expref: 922 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.738835:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517245880/2, rc 0, transno 4294975640 00000100:00100000:1.0:1783520935.738838:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993068928:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.738845:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517245880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993068416:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.738853:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025274 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x562:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.738860:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025274 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x562:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.738865:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025274 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x562:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.738871:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993069440 00010000:00010000:1.0:1783520935.738872:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025274 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x562:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.738878:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993069440 00010000:00010000:1.0:1783520935.738880:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025274 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x562:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.738882:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025290 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x564:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.738902:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993069568 00000100:00100000:0.0:1783520935.738906:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993069568 00010000:00010000:3.0:1783520935.738913:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000082179657/0xd5401cbabc01f521 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x564:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efcd5 expref: 921 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.738916:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.738926:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x564:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.738933:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x564:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.738933:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44144 00010000:00010000:3.0:1783520935.738939:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025290 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x564:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.738943:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515344000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+75:3627:x1870152993069440:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.738963:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1381 count 1 00010000:00010000:3.0:1783520935.738976:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000082179657/0xd5401cbabc01f521 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x564:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efcd5 expref: 921 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.738977:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025297 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x565:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.739001:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000082179657/0xd5401cbabc01f521 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x564:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efcd5 expref: 921 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.739001:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515345c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+76:3627:x1870152993068544:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3289us (3363us total) trans 4294975641 rc 0/0 00010000:00010000:0.0:1783520935.739001:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000cfba001f/0xd5401cbabc01f528 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x565:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efce3 expref: 921 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.739022:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44142 00010000:00010000:0.0:1783520935.739024:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x565:0x0].0x0 since=37s v=(0 0) 00000100:00100000:3.0:1783520935.739032:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993069952:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.739032:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x565:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.739039:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025297 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x565:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.739046:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44145 00000100:00100000:1.0:1783520935.739059:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cba0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+75:3628:x1870152993069568:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.739067:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000cfba001f/0xd5401cbabc01f528 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x565:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efce3 expref: 921 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.739075:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1382 count 1 00000100:00100000:3.0:1783520935.739077:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c8729c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.739078:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000cfba001f/0xd5401cbabc01f528 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x565:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efce3 expref: 921 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.739089:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02529e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x566:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.739091:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515346a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993070080:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.739106:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000099c719ae/0xd5401cbabc01f52f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x566:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efcf1 expref: 921 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.739111:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x566:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.739114:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x566:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.739117:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02529e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x566:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.739122:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124700c0 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.739132:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000099c719ae/0xd5401cbabc01f52f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x566:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efcf1 expref: 921 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.739139:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000099c719ae/0xd5401cbabc01f52f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x566:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efcf1 expref: 921 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.739144:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517245f80/2, rc 0, transno 4294975641 00000100:00100000:1.0:1783520935.739155:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993070208:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.739161:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517245f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993068544:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.739176:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6840 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.739195:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993069824:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.739225:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.739248:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1783520935.739255:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993069696:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.739270:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993069824 00000100:00100000:0.0:1783520935.739275:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993069824 00000100:00100000:1.0:1783520935.739280:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.739303:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993070336 00000100:00100000:1.0:1783520935.739308:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993069696 00000100:00100000:0.0:1783520935.739308:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993070336 00000100:00100000:1.0:1783520935.739313:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993069696 00000100:00100000:0.0:1783520935.739340:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44146 00000100:00100000:1.0:1783520935.739346:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44147 00000100:00100000:0.0:1783520935.739351:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+78:3626:x1870152993069824:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.739375:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cba1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+78:3628:x1870152993070336:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.739386:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1385 count 1 00010000:00010000:1.0:1783520935.739399:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0252a5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x569:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.739409:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002e3c29ca/0xd5401cbabc01f53d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x569:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efd1b expref: 921 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.739414:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1384 count 1 00010000:00010000:1.0:1783520935.739415:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x569:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.739422:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x569:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.739429:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc0252b3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x568:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.739432:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0252a5 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x569:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.739459:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004bee5422/0xd5401cbabc01f54b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x568:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efd0d expref: 921 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.739472:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002e3c29ca/0xd5401cbabc01f53d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x569:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd1b expref: 921 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.739475:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x568:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.739482:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x568:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.739492:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc0252b3 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x568:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.739510:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002e3c29ca/0xd5401cbabc01f53d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x569:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd1b expref: 921 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.739529:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c6680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993070464:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.739534:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004bee5422/0xd5401cbabc01f54b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x568:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd0d expref: 921 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.739551:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004bee5422/0xd5401cbabc01f54b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x568:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd0d expref: 921 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.739578:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515347800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993070720:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.739583:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6cc0 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.739605:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025243 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x55d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.739608:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.739619:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025243 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x55d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.739628:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025243 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x55d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.739635:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025243 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x55d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.739646:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.739716:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515345880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+79:3628:x1870152993067392:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6071us (6155us total) trans 4294975642 rc 0/0 00000100:00100000:1.0:1783520935.739723:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44135 00010000:00010000:1.0:1783520935.739745:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc02524a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x55c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.739748:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943512426300/2, rc 0, transno 4294975642 00010000:00010000:1.0:1783520935.739759:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc02524a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x55c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.739764:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943512426300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993067392:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.739770:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc02524a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x55c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.739780:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc02524a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x55c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.739798:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.739809:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993070848:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.739845:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.739869:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993070848 00000100:00100000:3.0:1783520935.739875:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993070848 00000100:00100000:1.0:1783520935.739881:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515346680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+80:3625:x1870152993067264:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6191us (6276us total) trans 4294975643 rc 0/0 00000100:00100000:1.0:1783520935.739891:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44136 00000100:00100000:1.0:1783520935.739916:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44149 00000100:00100000:1.0:1783520935.739925:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515344380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+79:3628:x1870152993070848:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.739937:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1386 count 1 00010000:00010000:1.0:1783520935.739945:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0252ba lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x56a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.739953:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000098454fbf/0xd5401cbabc01f544 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x56a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efd29 expref: 921 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.739958:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x56a:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.739960:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x56a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.739963:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0252ba lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x56a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.739981:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000098454fbf/0xd5401cbabc01f544 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x56a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd29 expref: 921 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.739990:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000098454fbf/0xd5401cbabc01f544 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x56a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd29 expref: 921 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.740000:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993070976:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.740030:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6540 going to sleep for 11 seconds 00000004:00080000:1.0:1783520935.740062:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435121a7b80/2, rc 0, transno 4294975643 00000100:00100000:1.0:1783520935.740074:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993067264:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.740124:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993071104:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.740145:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.740183:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993071104 00000100:00100000:0.0:1783520935.740187:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993071104 00000100:00100000:0.0:1783520935.740210:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44150 00000100:00100000:0.0:1783520935.740216:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+80:3625:x1870152993071104:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.740225:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1387 count 1 00010000:00010000:0.0:1783520935.740233:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0252c1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x56b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.740242:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000090f27ce1/0xd5401cbabc01f552 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x56b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efd37 expref: 921 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.740247:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x56b:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.740249:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x56b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.740252:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0252c1 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x56b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.740264:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000090f27ce1/0xd5401cbabc01f552 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x56b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd37 expref: 921 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.740274:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000090f27ce1/0xd5401cbabc01f552 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x56b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd37 expref: 921 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.740281:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515345180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993071232:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.740311:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470a80 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.740536:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993069952:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.740543:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515346a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993070080:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.740556:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025290 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x564:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.740560:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025297 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x565:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.740618:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.740646:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.740678:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.740703:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002371968 00000100:00100000:1.0:1783520935.740709:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002371968 00000100:00100000:0.0:1783520935.740725:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35736 00010000:00010000:1.0:1783520935.740729:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000cfba001f/0xd5401cbabc01f528 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x565:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efce3 expref: 924 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.740740:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+924:2387:x1870153002371840:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.740752:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.740761:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000082179657/0xd5401cbabc01f521 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x564:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efcd5 expref: 924 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.740771:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35738 00000100:00100000:1.0:1783520935.740788:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517247480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+924:2388:x1870153002371968:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.740788:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000082179657/0xd5401cbabc01f521 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x564:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efcd5 expref: 924 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.740805:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.740815:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000cfba001f/0xd5401cbabc01f528 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x565:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efce3 expref: 924 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.740824:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000082179657/0xd5401cbabc01f521 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x564:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efcd5 expref: 924 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.740836:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.740841:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000cfba001f/0xd5401cbabc01f528 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x565:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efce3 expref: 924 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.740852:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x564:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.740858:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x564:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.740864:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cfba001f/0xd5401cbabc01f528 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x565:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efce3 expref: 923 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.740868:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025290 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x564:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.740893:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025290 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x564:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.740894:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x565:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.740904:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x565:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.740915:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025297 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x565:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.740919:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.740933:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.740940:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025297 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x565:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.740966:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.740968:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+921:2387:x1870153002371840:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 231us (349us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.740975:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35736 00000100:00100000:0.0:1783520935.740991:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002372224 00000100:00100000:1.0:1783520935.740998:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517247480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+920:2388:x1870153002371968:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 212us (317us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.740998:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002372224 00000100:00100000:1.0:1783520935.741010:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35738 00010000:00010000:0.0:1783520935.741018:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002e3c29ca/0xd5401cbabc01f53d lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x569:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd1b expref: 920 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.741046:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35739 00000100:00100000:0.0:1783520935.741056:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517244380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+920:2387:x1870153002372224:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.741065:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.741072:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002e3c29ca/0xd5401cbabc01f53d lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x569:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd1b expref: 920 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.741085:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c6680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993070464:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.741086:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002e3c29ca/0xd5401cbabc01f53d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x569:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd1b expref: 920 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.741099:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.741108:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0252a5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x569:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.741119:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002e3c29ca/0xd5401cbabc01f53d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x569:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd1b expref: 920 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.741141:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x569:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.741144:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993070976:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.741150:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x569:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.741157:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0252a5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x569:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.741158:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0252ba lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x56a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.741185:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0252a5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x569:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.741190:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002372480 00000100:00100000:1.0:1783520935.741194:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002372480 00010000:00010000:1.0:1783520935.741211:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004bee5422/0xd5401cbabc01f54b lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x568:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd0d expref: 919 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.741214:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.741247:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35741 00000100:00100000:0.0:1783520935.741256:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517244380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+919:2387:x1870153002372224:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 201us (424us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.741261:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517246d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+919:2389:x1870153002372480:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.741274:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35739 00010000:00010000:1.0:1783520935.741276:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.741284:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004bee5422/0xd5401cbabc01f54b lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x568:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd0d expref: 918 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.741297:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004bee5422/0xd5401cbabc01f54b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x568:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd0d expref: 917 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.741305:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.741307:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025290 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x564:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.741325:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025290 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x564:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.741342:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x568:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.741347:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x568:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.741347:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025290 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x564:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.741382:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc0252b3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x568:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.741394:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.741397:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025290 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x564:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.741409:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc0252b3 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x568:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.741422:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.741441:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.741455:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002372608 00000100:00100000:3.0:1783520935.741459:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002372608 00000100:00100000:1.0:1783520935.741474:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517246d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+918:2389:x1870153002372480:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 216us (380us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.741476:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000098454fbf/0xd5401cbabc01f544 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x56a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd29 expref: 918 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.741490:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35741 00000100:00100000:1.0:1783520935.741522:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002372736 00000100:00100000:1.0:1783520935.741526:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002372736 00000100:00100000:0.0:1783520935.741536:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+82:3626:x1870152993069312:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2751us (2860us total) trans 4294975644 rc 0/0 00010000:00010000:1.0:1783520935.741543:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000090f27ce1/0xd5401cbabc01f552 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x56b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd37 expref: 918 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.741547:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44143 00000100:00100000:1.0:1783520935.741582:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35742 00000100:00100000:1.0:1783520935.741595:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517244a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+918:2390:x1870153002372736:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.741604:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.741606:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35743 00010000:00010000:1.0:1783520935.741614:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000090f27ce1/0xd5401cbabc01f552 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x56b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd37 expref: 918 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.741620:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357f450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+918:2387:x1870153002372608:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.741632:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.741642:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000098454fbf/0xd5401cbabc01f544 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x56a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd29 expref: 918 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.741643:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000090f27ce1/0xd5401cbabc01f552 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x56b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd37 expref: 918 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.741644:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025290 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x564:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.741667:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000098454fbf/0xd5401cbabc01f544 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x56a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd29 expref: 918 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.741679:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025290 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x564:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.741680:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x56b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.741688:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x56b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.741693:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000098454fbf/0xd5401cbabc01f544 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x56a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd29 expref: 918 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.741695:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025290 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x564:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.741697:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0252c1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.741699:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cba0700/2, rc 0, transno 4294975645 00000100:00100000:3.0:1783520935.741711:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993069696:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.741719:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x56a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.741724:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x56a:0x0].0x0 since=0s v=(0 0) 00000004:00080000:3.0:1783520935.741725:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351510b480/2, rc 0, transno 4294975644 00010000:00010000:1.0:1783520935.741725:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0252c1 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x56b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.741730:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0252ba lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.741731:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:3.0:1783520935.741733:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351510b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993069312:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.741739:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0252ac lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x567:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.741752:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0252ba lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x56a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.741756:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.741759:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.741763:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0252ac lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x567:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.741772:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0252ac lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x567:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.741775:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.741787:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0252ac lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x567:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.741788:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517244a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+917:2390:x1870153002372736:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 195us (430us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.741803:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357f450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+917:2387:x1870153002372608:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 186us (556us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.741806:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0252ac lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x567:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.741808:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35742 00000100:00100000:0.0:1783520935.741823:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35743 00010000:00010000:1.0:1783520935.741840:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0252a5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x569:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.741841:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.741844:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.741848:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.741853:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0252a5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x569:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.741863:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0252a5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x569:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.741872:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0252a5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x569:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.741876:0:35275:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004bee5422/0xd5401cbabc01f54b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x568:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd0d expref: 915 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.741895:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993071360:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.741895:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515347800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993070720:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.741899:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.741902:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.741908:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0252a5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x569:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.741912:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc0252b3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x568:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.741914:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.741920:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0252a5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x569:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.741923:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc0252b3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x568:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.741929:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0252a5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x569:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.741933:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.741943:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc0252b3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x568:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.741949:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0252a5 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x569:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.741950:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc0252b3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x568:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.741968:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993071360 00010000:00010000:2.0:1783520935.741970:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0252a5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x569:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.741971:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc0252b3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x568:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.741974:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993071360 00000100:00100000:1.0:1783520935.741992:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cba1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+82:3628:x1870152993070336:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2635us (2734us total) trans 4294975646 rc 0/0 00000100:00100000:1.0:1783520935.742003:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44147 00010000:00010000:0.0:1783520935.742003:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.742005:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.742011:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc0252b3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x568:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.742015:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44151 00010000:00010000:2.0:1783520935.742020:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.742023:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc0252b3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x568:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.742031:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517244e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+81:3626:x1870152993071360:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.742032:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc0252b3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x568:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.742036:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0252ba lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.742051:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc0252b3 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x568:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520935.742052:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1388 count 1 00010000:00010000:1.0:1783520935.742053:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0252ba lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.742063:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc0252b3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x568:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.742070:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc0252c8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x56c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.742075:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+82:3626:x1870152993069824:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2726us (2850us total) trans 4294975647 rc 0/0 00010000:00010000:2.0:1783520935.742083:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.742086:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.742089:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.742093:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44146 00010000:00010000:1.0:1783520935.742096:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0252ba lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.742098:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.742102:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000038963414/0xd5401cbabc01f559 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x56c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efd45 expref: 914 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.742110:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993071488:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.742110:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0252ba lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x56a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.742122:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x56c:0x0].0x0 since=37s v=(0 0) 00000004:00080000:0.0:1783520935.742122:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cba3480/2, rc 0, transno 4294975646 00010000:00010000:3.0:1783520935.742130:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x56c:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.742134:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993070336:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.742136:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.742137:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc0252c8 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x56c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.742146:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.742171:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515347800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993071616:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.742173:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000038963414/0xd5401cbabc01f559 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x56c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd45 expref: 914 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.742176:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.742182:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0252ba lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x56a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.742190:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.742195:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0252ba lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x56a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.742196:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000038963414/0xd5401cbabc01f559 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x56c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd45 expref: 914 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.742196:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.742213:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0252ba lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x56a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.742222:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517245c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993071744:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.742224:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0252ba lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x56a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.742225:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cba1500/2, rc 0, transno 4294975648 00000100:00100000:1.0:1783520935.742230:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515344380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+82:3628:x1870152993070848:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2303us (2385us total) trans 4294975648 rc 0/0 00000100:00100000:0.0:1783520935.742235:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993070848:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.742248:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0252ba lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x56a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.742249:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44149 00010000:00010000:2.0:1783520935.742272:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.742275:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.742279:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.742279:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993071488 00000100:00100000:0.0:1783520935.742284:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993071616 00000100:00100000:1.0:1783520935.742285:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993071488 00000100:00100000:3.0:1783520935.742286:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872f00 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.742287:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.742288:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993071616 00010000:00010000:2.0:1783520935.742299:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02529e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x566:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.742305:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02529e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x566:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.742310:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02529e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x566:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.742315:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02529e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x566:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.742323:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44152 00010000:00010000:2.0:1783520935.742326:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.742331:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44153 00000100:00100000:1.0:1783520935.742334:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363566c850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+83:3627:x1870152993071488:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.742345:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515346a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+83:3628:x1870152993071616:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.742349:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1389 count 1 00010000:00010000:1.0:1783520935.742387:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0252cf lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x56d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.742395:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1390 count 1 00010000:00010000:1.0:1783520935.742401:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000048ee2420/0xd5401cbabc01f560 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x56d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efd53 expref: 914 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.742410:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x56d:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.742411:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0252d6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x56e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.742416:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x56d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.742427:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0252cf lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x56d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.742432:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001b153684/0xd5401cbabc01f567 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x56e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efd61 expref: 914 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.742436:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cba3100/2, rc 0, transno 4294975647 00000100:00100000:2.0:1783520935.742449:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cba0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+84:3628:x1870152993069568:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3393us (3628us total) trans 4294975649 rc 0/0 00000100:00100000:3.0:1783520935.742453:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993069824:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.742454:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x56e:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.742463:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x56e:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.742466:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1783520935.742467:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44145 00010000:00010000:0.0:1783520935.742467:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0252d6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x56e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.742479:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515345f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993071872:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.742481:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000048ee2420/0xd5401cbabc01f560 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x56d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd53 expref: 914 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.742485:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.742491:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02529e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x566:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.742511:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000048ee2420/0xd5401cbabc01f560 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x56d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd53 expref: 914 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.742512:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001b153684/0xd5401cbabc01f567 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x56e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd61 expref: 914 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.742523:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.742529:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.742535:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02529e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x566:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.742542:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993072000:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.742544:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001b153684/0xd5401cbabc01f567 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x56e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd61 expref: 914 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.742553:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02529e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x566:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.742572:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517247480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993072256:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.742585:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515345500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993072128:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.742586:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02529e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x566:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.742589:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6600 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.742599:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02529e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x566:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.742607:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.742609:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993071872 00000100:00100000:1.0:1783520935.742616:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993071872 00000100:00100000:0.0:1783520935.742616:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.742629:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.742634:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.742637:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.742638:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993072256 00000100:00100000:0.0:1783520935.742642:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993072256 00000100:00100000:1.0:1783520935.742653:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44154 00000100:00100000:1.0:1783520935.742664:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517245500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+85:3626:x1870152993071872:12345-0@lo:6:osp-syn-1-0.0 00000004:00080000:2.0:1783520935.742666:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cba2a00/2, rc 0, transno 4294975649 00000100:00100000:0.0:1783520935.742672:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44155 00000100:00100000:2.0:1783520935.742680:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993069568:0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.742683:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1391 count 1 00000100:00100000:0.0:1783520935.742684:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517244700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+85:3626:x1870152993072256:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.742700:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0252dd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x56f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.742702:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1392 count 1 00010000:00010000:0.0:1783520935.742713:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0252e4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x570:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.742718:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000056ae370c/0xd5401cbabc01f56e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x56f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efd6f expref: 914 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.742726:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c1dd363f/0xd5401cbabc01f575 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x570:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efd7d expref: 914 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.742736:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x56f:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.742737:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x570:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.742742:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x56f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.742742:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x570:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.742750:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517244380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993072384:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.742750:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0252e4 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x570:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.742752:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0252dd lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x56f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.742780:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c1dd363f/0xd5401cbabc01f575 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x570:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd7d expref: 914 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.742786:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.742789:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000056ae370c/0xd5401cbabc01f56e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x56f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd6f expref: 914 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.742798:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c1dd363f/0xd5401cbabc01f575 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x570:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd7d expref: 914 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.742806:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000056ae370c/0xd5401cbabc01f56e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x56f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd6f expref: 914 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.742812:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993072384 00000100:00100000:3.0:1783520935.742818:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993072384 00000100:00100000:0.0:1783520935.742833:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515347480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993072512:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.742835:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c6680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993072640:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.742852:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44156 00000100:00100000:3.0:1783520935.742862:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cba3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+86:3627:x1870152993072384:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520935.742878:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1393 count 1 00000100:00100000:0.0:1783520935.742879:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470000 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.742881:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6300 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.742917:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025297 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x565:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.742926:0:35282:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000090f27ce1/0xd5401cbabc01f552 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x56b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd37 expref: 914 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.742929:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025297 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x565:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.742948:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515345180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993071232:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.742949:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025297 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x565:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.742961:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0252eb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x571:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.742961:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025297 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x565:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.742965:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0252c1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.742987:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a85a679f/0xd5401cbabc01f57c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x571:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efd8b expref: 913 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.742991:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0252c1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.742994:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x571:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.742997:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.743002:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x571:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.743006:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.743013:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0252eb lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x571:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.743013:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0252c1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.743014:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025297 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x565:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.743036:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfbc00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.743037:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0252c1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.743042:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025297 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x565:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.743051:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0252c1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x56b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.743059:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025297 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x565:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.743073:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.743075:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025297 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x565:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.743082:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515344000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+87:3627:x1870152993069440:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4140us (4286us total) trans 4294975650 rc 0/0 00010000:00010000:2.0:1783520935.743088:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025297 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x565:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.743092:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a85a679f/0xd5401cbabc01f57c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x571:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd8b expref: 913 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.743096:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44144 00010000:00010000:3.0:1783520935.743112:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a85a679f/0xd5401cbabc01f57c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x571:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd8b expref: 913 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.743118:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.743125:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0252c1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x56b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.743131:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517246d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993072768:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.743138:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfa600 already unused, calling callback (ffffffffc0bf9e90) 00000004:00080000:1.0:1783520935.743140:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513b05f80/2, rc 0, transno 4294975650 00010000:00010000:2.0:1783520935.743144:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0252c1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x56b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.743151:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+87:3625:x1870152993071104:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2934us (3003us total) trans 4294975651 rc 0/0 00000100:00100000:1.0:1783520935.743154:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b05f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993069440:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.743162:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0252c1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x56b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.743166:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44150 00010000:00010000:2.0:1783520935.743180:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0252c1 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x56b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.743189:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0252c1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x56b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.743208:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.743212:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.743214:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993072896:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.743217:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.743227:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520935.743238:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cba2300/2, rc 0, transno 4294975651 00000100:00100000:0.0:1783520935.743242:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.743253:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993071104:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.743267:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872b40 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.743304:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993073024:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.743308:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993072896 00000100:00100000:2.0:1783520935.743313:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993072896 00000100:00100000:1.0:1783520935.743337:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.743345:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44157 00000100:00100000:0.0:1783520935.743354:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993073024 00000100:00100000:2.0:1783520935.743355:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515345180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+87:3628:x1870152993072896:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.743387:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993073024 00002000:00080000:2.0:1783520935.743396:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1394 count 1 00010000:00010000:2.0:1783520935.743409:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0252f2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x572:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.743422:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000071d09fed/0xd5401cbabc01f704 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x572:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efd92 expref: 913 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.743425:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44158 00010000:00010000:2.0:1783520935.743435:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x572:0x0].0x0 since=37s v=(0 0) 00000100:00100000:0.0:1783520935.743438:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+88:3625:x1870152993073024:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.743441:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x572:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.743448:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0252f2 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x572:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.743455:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1395 count 1 00010000:00010000:0.0:1783520935.743471:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0252f9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x573:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.743472:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000071d09fed/0xd5401cbabc01f704 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x572:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd92 expref: 913 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.743476:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.743511:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002372864 00010000:00010000:2.0:1783520935.743512:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000071d09fed/0xd5401cbabc01f704 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x572:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd92 expref: 913 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.743512:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008f6a9c72/0xd5401cbabc01f583 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x573:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efd99 expref: 913 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.743517:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002372864 00000100:00100000:3.0:1783520935.743520:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517245c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993071744:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.743532:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x573:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.743537:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000038963414/0xd5401cbabc01f559 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x56c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd45 expref: 914 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.743538:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x573:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.743546:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0252f9 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x573:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.743562:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993073152:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.743573:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35744 00010000:00010000:0.0:1783520935.743578:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008f6a9c72/0xd5401cbabc01f583 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x573:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd99 expref: 914 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.743585:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943512427800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+914:2387:x1870153002372864:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.743595:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008f6a9c72/0xd5401cbabc01f583 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x573:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd99 expref: 914 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.743599:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520935.743609:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f000 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.743609:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000038963414/0xd5401cbabc01f559 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x56c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd45 expref: 914 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.743612:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993073280:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.743622:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc0252c8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x56c:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.743633:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000038963414/0xd5401cbabc01f559 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x56c:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd45 expref: 914 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.743650:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124700c0 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.743652:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000038963414/0xd5401cbabc01f559 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x56c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd45 expref: 914 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.743666:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x56c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.743670:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x56c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.743677:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc0252c8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.743692:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc0252c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x56c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.743706:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515345500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993072128:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.743718:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.743721:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0252d6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x56e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.743729:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc0252c8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.743740:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc0252c8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.743748:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc0252c8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.743756:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc0252c8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x56c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.743771:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.743783:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.743789:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc0252c8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x56c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.743795:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.743798:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc0252c8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x56c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.743803:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943512427800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+913:2387:x1870153002372864:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 220us (327us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.743807:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc0252c8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x56c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.743819:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35744 00010000:00010000:0.0:1783520935.743822:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc0252c8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x56c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.743828:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc0252c8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x56c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.743844:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.743847:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.743850:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.743864:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517244e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+89:3626:x1870152993071360:12345-0@lo:6:osp-syn-1-0.0 Request processed in 1838us (1934us total) trans 4294975652 rc 0/0 00000100:00100000:2.0:1783520935.743872:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44151 00000004:00080000:1.0:1783520935.743873:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cba0000/2, rc 0, transno 4294975652 00000100:00100000:1.0:1783520935.743884:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993071360:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.743934:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943512424a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993073408:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.743961:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.743987:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993073408 00000100:00100000:2.0:1783520935.743991:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993073408 00000100:00100000:2.0:1783520935.744017:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44159 00000100:00100000:2.0:1783520935.744025:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+89:3626:x1870152993073408:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.744037:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1396 count 1 00010000:00010000:2.0:1783520935.744049:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025300 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x574:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.744061:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc01f70b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x574:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efda0 expref: 912 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.744068:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x574:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.744072:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x574:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.744077:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025300 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x574:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.744087:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515347480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993072512:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.744100:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc01f70b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x574:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efda0 expref: 912 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.744102:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0252e4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x570:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.744127:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc01f70b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x574:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efda0 expref: 912 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.744138:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cc7fb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993073536:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.744170:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f180 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.744255:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c6680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993072640:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.744272:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0252dd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x56f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.744280:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993072000:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.744297:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0252cf lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x56d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.744389:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517246d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993072768:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.744403:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0252eb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x571:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.744551:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.744577:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.744590:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002373120 00000100:00100000:0.0:1783520935.744596:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002373120 00000100:00100000:3.0:1783520935.744600:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002373376 00000100:00100000:3.0:1783520935.744606:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002373376 00000100:00100000:2.0:1783520935.744606:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.744615:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c1dd363f/0xd5401cbabc01f575 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x570:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd7d expref: 913 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.744635:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.744645:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002372992 00000100:00100000:0.0:1783520935.744650:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002372992 00010000:00010000:3.0:1783520935.744665:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000048ee2420/0xd5401cbabc01f560 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x56d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd53 expref: 915 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.744665:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001b153684/0xd5401cbabc01f567 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x56e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd61 expref: 915 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.744667:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002373248 00000100:00100000:1.0:1783520935.744673:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002373248 00000100:00100000:0.0:1783520935.744694:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35745 00010000:00010000:1.0:1783520935.744696:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000056ae370c/0xd5401cbabc01f56e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x56f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd6f expref: 916 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.744703:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35746 00000100:00100000:0.0:1783520935.744707:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cc7df80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+916:2387:x1870153002373120:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.744709:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.744714:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cc7ca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+916:2387:x1870153002372992:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.744721:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.744723:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.744727:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c1dd363f/0xd5401cbabc01f575 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x570:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd7d expref: 916 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.744728:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001b153684/0xd5401cbabc01f567 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x56e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd61 expref: 916 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.744740:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001b153684/0xd5401cbabc01f567 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x56e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd61 expref: 916 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.744740:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c1dd363f/0xd5401cbabc01f575 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x570:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd7d expref: 916 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.744744:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002373504 00000100:00100000:1.0:1783520935.744749:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002373504 00000100:00100000:2.0:1783520935.744759:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35747 00010000:00010000:1.0:1783520935.744767:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a85a679f/0xd5401cbabc01f57c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x571:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd8b expref: 917 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.744768:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c1dd363f/0xd5401cbabc01f575 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x570:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd7d expref: 917 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.744773:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cc7c700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+917:2387:x1870153002373376:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.744789:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.744796:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x570:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.744800:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000048ee2420/0xd5401cbabc01f560 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x56d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd53 expref: 916 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.744802:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001b153684/0xd5401cbabc01f567 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x56e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd61 expref: 916 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.744802:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x570:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.744810:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35748 00010000:00010000:0.0:1783520935.744811:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0252e4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x570:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.744825:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x56e:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.744825:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cc7ea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+915:2387:x1870153002373248:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.744828:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0252e4 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x570:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.744831:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000048ee2420/0xd5401cbabc01f560 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x56d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd53 expref: 915 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.744832:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x56e:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.744839:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0252d6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.744841:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.744852:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000056ae370c/0xd5401cbabc01f56e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x56f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd6f expref: 915 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.744856:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.744860:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0252d6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x56e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.744868:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000048ee2420/0xd5401cbabc01f560 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x56d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd53 expref: 915 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.744875:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000056ae370c/0xd5401cbabc01f56e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x56f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd6f expref: 915 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.744883:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cc7df80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+915:2387:x1870153002373120:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 178us (329us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.744889:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.744897:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x56d:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.744902:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35745 00010000:00010000:2.0:1783520935.744904:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x56d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.744906:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000056ae370c/0xd5401cbabc01f56e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x56f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd6f expref: 914 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.744908:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cc7ca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+914:2387:x1870153002372992:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 196us (303us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.744915:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0252cf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.744921:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35749 00010000:00010000:1.0:1783520935.744931:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x56f:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.744932:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35746 00000100:00100000:0.0:1783520935.744932:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357d850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+912:2387:x1870153002373504:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.744934:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0252cf lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x56d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.744937:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x56f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.744944:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0252dd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.744946:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.744953:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0252d6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.744955:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0252dd lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x56f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.744955:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a85a679f/0xd5401cbabc01f57c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x571:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd8b expref: 911 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.744963:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.744966:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0252d6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.744978:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a85a679f/0xd5401cbabc01f57c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x571:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd8b expref: 911 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.744982:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0252d6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.744991:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cc7c700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+911:2387:x1870153002373376:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 221us (411us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.744993:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0252d6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x56e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.744993:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.745001:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a85a679f/0xd5401cbabc01f57c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x571:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd8b expref: 911 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.745010:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35747 00010000:00010000:0.0:1783520935.745016:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x571:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.745023:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cc7ea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+909:2387:x1870153002373248:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 199us (384us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.745023:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x571:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.745031:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0252eb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x571:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.745033:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520935.745037:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35748 00010000:00010000:2.0:1783520935.745041:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0252d6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x56e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.745050:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0252eb lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x571:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.745055:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfa000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.745061:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0252d6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x56e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.745066:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to regular blp list, 0 blwis in pool 00010000:00010000:1.0:1783520935.745068:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0252dd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.745073:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.745075:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0252d6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x56e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.745085:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0252dd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.745094:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0252d6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x56e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.745098:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0252dd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.745098:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357d850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+908:2387:x1870153002373504:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 164us (393us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.745116:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0252d6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x56e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.745116:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35749 00010000:00010000:1.0:1783520935.745118:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0252dd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x56f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.745143:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.745147:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.745151:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0252dd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x56f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.745158:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.745158:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0252e4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x570:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.745169:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0252e4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x570:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.745179:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0252dd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x56f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.745180:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0252e4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x570:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.745205:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515346a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+91:3628:x1870152993071616:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2864us (3007us total) trans 4294975653 rc 0/0 00010000:00010000:2.0:1783520935.745205:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0252dd lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x56f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.745205:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0252e4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x570:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.745219:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44153 00000100:00100000:1.0:1783520935.745232:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517245500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+90:3626:x1870152993071872:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2567us (2707us total) trans 4294975654 rc 0/0 00010000:00010000:0.0:1783520935.745232:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.745238:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0252dd lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x56f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.745245:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44154 00000004:00080000:3.0:1783520935.745253:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515347800/2, rc 0, transno 4294975653 00010000:00010000:2.0:1783520935.745253:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0252dd lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x56f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.745267:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515347800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993071616:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.745278:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.745283:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0252e4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x570:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.745287:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515345f80/2, rc 0, transno 4294975654 00010000:00010000:2.0:1783520935.745295:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfa400 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520935.745296:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515345f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993071872:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.745299:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0252e4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x570:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.745310:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517244700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+90:3626:x1870152993072256:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2627us (2704us total) trans 4294975655 rc 0/0 00010000:00010000:2.0:1783520935.745311:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0252e4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x570:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.745324:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44155 00010000:00010000:2.0:1783520935.745330:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0252e4 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x570:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.745339:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0252e4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x570:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.745398:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517247480/2, rc 0, transno 4294975655 00010000:00010000:2.0:1783520935.745407:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.745407:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517247480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993072256:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.745412:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.745417:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.745418:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:2.0:1783520935.745425:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.745431:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.745432:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517246d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993073920:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.745436:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517244a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993073792:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.745436:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.745449:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0252eb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x571:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.745460:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0252eb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x571:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.745468:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.745473:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0252eb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x571:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.745474:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.745476:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002373632 00010000:00010000:2.0:1783520935.745485:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0252eb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x571:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.745513:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943512427800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993073664:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.745514:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.745544:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.745545:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002373632 00000100:00100000:3.0:1783520935.745572:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.745574:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993073920 00000100:00100000:1.0:1783520935.745579:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993073920 00000100:00100000:1.0:1783520935.745607:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993073664 00000100:00100000:0.0:1783520935.745609:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.745613:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993073664 00000100:00100000:2.0:1783520935.745619:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cba3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+92:3627:x1870152993072384:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2758us (2834us total) trans 4294975656 rc 0/0 00000100:00100000:2.0:1783520935.745627:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44156 00000100:00100000:1.0:1783520935.745636:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993073792 00000100:00100000:1.0:1783520935.745640:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993073792 00000100:00100000:2.0:1783520935.745648:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44160 00010000:00010000:3.0:1783520935.745657:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008f6a9c72/0xd5401cbabc01f583 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x573:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd99 expref: 908 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.745660:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+92:3626:x1870152993073920:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.745669:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520935.745671:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44161 00010000:00010000:0.0:1783520935.745675:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0252eb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x571:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.745682:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1399 count 1 00000100:00100000:1.0:1783520935.745684:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+92:3627:x1870152993073664:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.745688:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002373760 00010000:00010000:0.0:1783520935.745692:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.745698:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0252eb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x571:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.745699:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025307 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x577:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.745703:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1397 count 1 00010000:00010000:0.0:1783520935.745718:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0252eb lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x571:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.745719:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc02530e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x575:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.745725:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000fa7c7ff4/0xd5401cbabc01f591 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x577:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efdb5 expref: 908 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.745726:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002373760 00010000:00010000:3.0:1783520935.745743:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000071d09fed/0xd5401cbabc01f704 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x572:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd92 expref: 909 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.745744:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0252eb lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x571:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.745751:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001bd728f8/0xd5401cbabc01f58a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x575:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efda7 expref: 909 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.745766:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0252eb lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x571:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.745771:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x577:0x0].0x0 since=37s v=(0 0) 00000100:00100000:3.0:1783520935.745774:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002373888 00010000:00010000:1.0:1783520935.745777:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x575:0x0].0x0 since=37s v=(0 0) 00000100:00100000:3.0:1783520935.745778:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002373888 00010000:00010000:2.0:1783520935.745782:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x577:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.745786:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x575:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.745793:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc01f70b lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x574:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efda0 expref: 910 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.745794:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025307 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x577:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.745795:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc02530e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x575:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.745796:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.745799:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.745802:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.745815:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35750 00000100:00100000:3.0:1783520935.745826:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35751 00000100:00100000:0.0:1783520935.745827:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515347480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+910:2389:x1870153002373632:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520935.745833:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515345500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+910:2387:x1870153002373760:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.745835:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.745840:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.745841:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008f6a9c72/0xd5401cbabc01f583 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x573:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd99 expref: 910 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.745845:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000fa7c7ff4/0xd5401cbabc01f591 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x577:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdb5 expref: 910 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.745845:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001bd728f8/0xd5401cbabc01f58a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x575:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efda7 expref: 910 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.745847:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000071d09fed/0xd5401cbabc01f704 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x572:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd92 expref: 910 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.745858:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008f6a9c72/0xd5401cbabc01f583 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x573:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd99 expref: 910 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.745875:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000071d09fed/0xd5401cbabc01f704 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x572:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd92 expref: 910 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.745881:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001bd728f8/0xd5401cbabc01f58a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x575:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efda7 expref: 910 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.745882:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000fa7c7ff4/0xd5401cbabc01f591 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x577:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdb5 expref: 910 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.745882:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x573:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.745890:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x573:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.745899:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0252f9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x573:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.745908:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x572:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.745912:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c7b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993074176:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.745914:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x572:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.745918:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0252f9 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x573:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.745920:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cc7ce00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993074048:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.745922:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0252f2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x572:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.745933:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0252f2 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x572:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.745939:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.745946:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.745955:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6f00 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.745960:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943515746cc0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.745967:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515345500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+910:2387:x1870153002373760:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 133us (492us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.745967:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515347480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+910:2389:x1870153002373632:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 140us (535us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.745975:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35750 00000004:00080000:1.0:1783520935.745977:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517244380/2, rc 0, transno 4294975656 00000100:00100000:3.0:1783520935.745981:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35751 00000100:00100000:1.0:1783520935.745988:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517244380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993072384:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.745995:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35752 00000100:00100000:0.0:1783520935.746004:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514f9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+908:2387:x1870153002373888:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.746014:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520935.746019:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44162 00010000:00010000:0.0:1783520935.746020:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc01f70b lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x574:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efda0 expref: 908 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.746027:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517245c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+92:3628:x1870152993073792:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.746031:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc01f70b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x574:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efda0 expref: 908 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.746035:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993074304:0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520935.746040:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1398 count 1 00010000:00010000:0.0:1783520935.746051:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x574:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.746054:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025315 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x576:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.746054:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x574:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.746059:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025300 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x574:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.746065:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000099d089f6/0xd5401cbabc01f712 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x576:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efdae expref: 908 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.746066:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025300 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x574:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.746074:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.746078:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x576:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.746083:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000071d09fed/0xd5401cbabc01f704 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x572:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd92 expref: 908 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.746084:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x576:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.746085:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.746090:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025315 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x576:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.746102:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993073152:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.746114:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993074304 00000100:00100000:0.0:1783520935.746114:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514f9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+907:2387:x1870153002373888:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 110us (505us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.746118:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993074304 00010000:00010000:2.0:1783520935.746119:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0252f2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x572:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.746129:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35752 00010000:00010000:2.0:1783520935.746134:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0252f2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x572:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.746144:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44163 00000100:00100000:1.0:1783520935.746154:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+93:3626:x1870152993074304:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.746167:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0252f2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x572:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.746169:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008f6a9c72/0xd5401cbabc01f583 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x573:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efd99 expref: 906 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.746170:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1400 count 1 00010000:00010000:3.0:1783520935.746175:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000099d089f6/0xd5401cbabc01f712 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x576:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdae expref: 906 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.746183:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000066426cc1/0xd5401cbabc02531c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x578:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.746193:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0252f2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x572:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.746197:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993073280:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.746199:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000099d089f6/0xd5401cbabc01f712 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x576:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdae expref: 905 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.746213:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001eb0b58f/0xd5401cbabc01f719 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x578:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efdbc expref: 905 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.746216:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0252f9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x573:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.746220:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0252f2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x572:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.746225:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e0f5f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993074432:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.746241:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x578:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.746241:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0252f9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x573:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.746248:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x578:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.746250:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520935.746254:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872f00 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.746257:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0252f9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x573:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.746259:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000066426cc1/0xd5401cbabc02531c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x578:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.746270:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0252f9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x573:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.746280:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0252f9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x573:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.746289:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001eb0b58f/0xd5401cbabc01f719 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x578:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdbc expref: 905 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.746296:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520935.746302:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001eb0b58f/0xd5401cbabc01f719 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x578:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdbc expref: 905 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.746317:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c6300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993074560:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.746342:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6300 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.746343:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515345180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+95:3628:x1870152993072896:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2990us (3100us total) trans 4294975657 rc 0/0 00000100:00100000:2.0:1783520935.746354:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44157 00010000:00010000:2.0:1783520935.746402:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520935.746407:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0252f2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x572:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.746415:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.746418:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0252f2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x572:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.746427:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0252f2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x572:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.746439:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0252f2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x572:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.746447:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0252f2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x572:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.746450:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+94:3625:x1870152993073024:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3014us (3112us total) trans 4294975658 rc 0/0 00000100:00100000:0.0:1783520935.746465:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44158 00010000:00010000:2.0:1783520935.746479:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.746484:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0252f9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x573:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.746487:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0252cf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.746489:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e1c4700/2, rc 0, transno 4294975657 00010000:00010000:2.0:1783520935.746511:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.746515:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0252cf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.746517:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0252f9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x573:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.746535:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0252cf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x56d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.746537:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0252f9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x573:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.746551:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993072896:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.746555:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0252cf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x56d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.746564:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993074688:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.746569:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0252f9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x573:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.746589:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0252f9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x573:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.746596:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.746601:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.746605:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.746612:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0252cf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x56d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.746613:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.746618:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.746623:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.746624:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.746630:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0252cf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x56d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.746634:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.746641:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0252cf lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x56d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.746641:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993074688 00000100:00100000:1.0:1783520935.746647:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993074688 00010000:00010000:3.0:1783520935.746659:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0252cf lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x56d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.746664:0:35276:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc01f70b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x574:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efda0 expref: 905 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.746679:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363566c850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+95:3627:x1870152993071488:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4348us (4535us total) trans 4294975659 rc 0/0 00000100:00100000:1.0:1783520935.746683:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44164 00010000:00010000:3.0:1783520935.746689:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0252cf lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x56d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.746695:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44152 00000100:00100000:1.0:1783520935.746698:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+95:3627:x1870152993074688:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.746710:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cc7fb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993073536:192.168.204.33@tcp:104: 00002000:00080000:1.0:1783520935.746719:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1401 count 1 00010000:00010000:3.0:1783520935.746721:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.746725:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.746728:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.746729:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025300 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x574:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.746733:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025323 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x579:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.746747:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025300 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x574:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.746751:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000011832fe6/0xd5401cbabc01f598 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x579:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efdc3 expref: 904 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.746766:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025300 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x574:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.746766:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x579:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.746769:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x579:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.746776:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025323 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x579:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.746783:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025300 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x574:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.746792:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025300 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x574:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.746801:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000011832fe6/0xd5401cbabc01f598 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x579:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdc3 expref: 904 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.746809:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cba2680/2, rc 0, transno 4294975658 00010000:00010000:1.0:1783520935.746813:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000011832fe6/0xd5401cbabc01f598 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x579:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdc3 expref: 904 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.746814:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.746826:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993073024:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.746832:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c5f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993074816:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.746840:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.746847:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025300 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x574:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.746855:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.746859:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025300 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x574:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.746864:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e0f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993074944:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.746867:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025300 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x574:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.746881:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025300 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x574:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.746883:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6600 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.746892:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025300 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x574:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.746899:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.746911:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.746914:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.746917:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520935.746919:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cba0380/2, rc 0, transno 4294975659 00000100:00100000:2.0:1783520935.746921:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+95:3626:x1870152993073408:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2894us (2956us total) trans 4294975660 rc 0/0 00000100:00100000:2.0:1783520935.746930:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44159 00000100:00100000:1.0:1783520935.746935:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993071488:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.746944:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993074944 00000100:00100000:2.0:1783520935.746949:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993074944 00000100:00100000:2.0:1783520935.746976:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44165 00000100:00100000:2.0:1783520935.746983:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e0f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+95:3628:x1870152993074944:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.746989:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993075072:0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.747000:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1402 count 1 00000004:00080000:1.0:1783520935.747000:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943512424a80/2, rc 0, transno 4294975660 00000100:00100000:1.0:1783520935.747013:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943512424a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993073408:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.747016:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02532a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x57a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.747021:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.747035:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000003387dfa/0xd5401cbabc022938 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x57a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3028 expref: 904 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.747043:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x57a:0x0].0x0 since=36s v=(0 0) 00000100:00100000:0.0:1783520935.747045:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:2.0:1783520935.747047:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x57a:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.747053:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02532a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x57a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.747054:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e0f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993075200:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.747082:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000003387dfa/0xd5401cbabc022938 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x57a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3028 expref: 904 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.747084:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.747097:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000003387dfa/0xd5401cbabc022938 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x57a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3028 expref: 904 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.747108:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993075072 00000100:00100000:2.0:1783520935.747110:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cc7fb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993075328:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.747114:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993075072 00000100:00100000:0.0:1783520935.747115:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993075200 00000100:00100000:0.0:1783520935.747120:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993075200 00000100:00100000:2.0:1783520935.747144:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f000 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.747147:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44166 00000100:00100000:0.0:1783520935.747155:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44167 00000100:00100000:1.0:1783520935.747157:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514f9b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+97:3625:x1870152993075072:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.747166:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514f99c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+97:3625:x1870152993075200:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.747169:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1403 count 1 00002000:00080000:0.0:1783520935.747181:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1404 count 1 00010000:00010000:1.0:1783520935.747182:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025331 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x57b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.747194:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ce9d266d/0xd5401cbabc01f59f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x57b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efdd1 expref: 904 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.747197:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025338 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x57c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.747199:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x57b:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.747205:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x57b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.747212:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025331 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x57b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.747219:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000020b97928/0xd5401cbabc01f727 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x57c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efdd8 expref: 904 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.747228:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x57c:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.747231:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x57c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.747234:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ce9d266d/0xd5401cbabc01f59f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x57b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdd1 expref: 904 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.747236:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025338 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x57c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.747264:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ce9d266d/0xd5401cbabc01f59f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x57b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdd1 expref: 904 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.747271:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000020b97928/0xd5401cbabc01f727 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x57c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdd8 expref: 904 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.747279:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c5500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993075456:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.747287:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000020b97928/0xd5401cbabc01f727 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x57c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdd8 expref: 904 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.747304:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9c000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993075584:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.747314:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6240 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.747333:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470e40 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.747476:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c7b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993074176:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.747491:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc02530e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x575:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.747533:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cc7ce00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993074048:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.747548:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025307 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x577:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.747610:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.747629:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002374016 00000100:00100000:3.0:1783520935.747633:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002374016 00000100:00100000:0.0:1783520935.747638:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.747677:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002374144 00000100:00100000:1.0:1783520935.747680:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002374144 00010000:00010000:1.0:1783520935.747695:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000fa7c7ff4/0xd5401cbabc01f591 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x577:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdb5 expref: 906 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.747698:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c6300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993074560:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.747715:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000066426cc1/0xd5401cbabc02531c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x578:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.747725:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35753 00000100:00100000:1.0:1783520935.747736:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515346d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+906:2390:x1870153002374144:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.747745:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.747750:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000fa7c7ff4/0xd5401cbabc01f591 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x577:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdb5 expref: 906 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.747761:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000fa7c7ff4/0xd5401cbabc01f591 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x577:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdb5 expref: 906 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.747769:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001bd728f8/0xd5401cbabc01f58a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x575:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efda7 expref: 906 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.747780:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000fa7c7ff4/0xd5401cbabc01f591 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x577:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdb5 expref: 906 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.747791:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x577:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.747794:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35754 00010000:00010000:1.0:1783520935.747794:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x577:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.747799:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025307 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x577:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.747802:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9e300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+905:2390:x1870153002374016:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.747807:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025307 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x577:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.747810:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520935.747812:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.747816:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001bd728f8/0xd5401cbabc01f58a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x575:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efda7 expref: 905 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.747827:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001bd728f8/0xd5401cbabc01f58a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x575:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efda7 expref: 905 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.747832:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.747838:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002374272 00000100:00100000:0.0:1783520935.747844:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002374272 00000100:00100000:1.0:1783520935.747864:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515346d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+906:2390:x1870153002374144:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 127us (222us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.747864:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001eb0b58f/0xd5401cbabc01f719 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x578:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdbc expref: 906 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.747881:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35753 00010000:00010000:3.0:1783520935.747891:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001bd728f8/0xd5401cbabc01f58a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x575:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efda7 expref: 906 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.747906:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35755 00000100:00100000:1.0:1783520935.747914:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cc7ce00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+905:2388:x1870153002374272:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.747923:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.747928:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001eb0b58f/0xd5401cbabc01f719 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x578:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdbc expref: 905 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.747939:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001eb0b58f/0xd5401cbabc01f719 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x578:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdbc expref: 905 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.747952:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e0f5f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993074432:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.747962:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001eb0b58f/0xd5401cbabc01f719 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x578:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdbc expref: 905 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.747966:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025315 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x576:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.747970:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x575:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.747974:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x578:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.747978:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x575:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.747980:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x578:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.747987:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc02530e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x575:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.747987:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000066426cc1/0xd5401cbabc02531c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x578:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.747997:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc02530e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x575:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.747997:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000066426cc1/0xd5401cbabc02531c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x578:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.748016:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.748016:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.748024:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc02530e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x575:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.748032:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc02530e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x575:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.748033:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000066426cc1/0xd5401cbabc02531c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x578:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.748041:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc02530e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x575:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.748045:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9e300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+903:2390:x1870153002374016:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 242us (433us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.748045:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cc7ce00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+903:2388:x1870153002374272:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 130us (234us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.748055:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000066426cc1/0xd5401cbabc02531c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x578:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.748061:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc02530e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x575:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.748065:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35755 00000100:00100000:3.0:1783520935.748076:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35754 00010000:00010000:2.0:1783520935.748080:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000066426cc1/0xd5401cbabc02531c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x578:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.748089:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025307 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x577:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.748096:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.748104:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000066426cc1/0xd5401cbabc02531c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x578:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.748106:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025307 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x577:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.748127:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025307 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x577:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.748133:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520935.748135:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025307 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x577:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.748152:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to regular blp list, 3 blwis in pool 00000100:00100000:0.0:1783520935.748162:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+98:3627:x1870152993073664:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2483us (2618us total) trans 4294975661 rc 0/0 00000100:00100000:0.0:1783520935.748167:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44161 00000100:00100000:2.0:1783520935.748226:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+99:3626:x1870152993074304:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2072us (2157us total) trans 4294975662 rc 0/0 00000100:00100000:2.0:1783520935.748235:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44163 00000004:00080000:0.0:1783520935.748238:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e1c6a00/2, rc 0, transno 4294975662 00000100:00100000:1.0:1783520935.748243:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+99:3626:x1870152993073920:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2585us (2775us total) trans 4294975663 rc 0/0 00000100:00100000:0.0:1783520935.748252:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993074304:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.748255:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00000100:00100000:1.0:1783520935.748255:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44160 00010000:00010000:2.0:1783520935.748264:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc02530e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x575:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.748272:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7200 already unused, calling callback (ffffffffc0bf9e90) 00000004:00080000:0.0:1783520935.748272:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517246d80/2, rc 0, transno 4294975663 00010000:00010000:2.0:1783520935.748277:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc02530e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x575:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.748280:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517246d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993073920:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.748293:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc02530e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x575:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.748298:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943512427800/2, rc 0, transno 4294975661 00000100:00100000:1.0:1783520935.748305:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943512427800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993073664:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.748308:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc02530e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x575:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.748317:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc02530e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x575:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.748333:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520935.748337:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000066426cc1/0xd5401cbabc02531c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x578:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.748345:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.748349:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000066426cc1/0xd5401cbabc02531c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x578:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.748394:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1783520935.748405:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e0f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993075712:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.748413:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000066426cc1/0xd5401cbabc02531c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x578:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.748426:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000066426cc1/0xd5401cbabc02531c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x578:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.748436:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000066426cc1/0xd5401cbabc02531c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x578:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.748437:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.748463:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.748467:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025307 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x577:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.748469:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e0f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993075840:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.748479:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.748480:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.748485:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025307 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x577:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.748494:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025307 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x577:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.748512:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.748524:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025307 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x577:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.748532:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025307 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x577:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.748548:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.748549:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.748552:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.748560:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.748569:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993075712 00000100:00100000:2.0:1783520935.748573:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993075712 00000100:00100000:2.0:1783520935.748599:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993075840 00000100:00100000:1.0:1783520935.748602:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c5f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993074816:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.748603:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993075840 00000100:00100000:0.0:1783520935.748617:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44168 00010000:00010000:1.0:1783520935.748620:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025323 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x579:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.748630:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515344700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+99:3628:x1870152993075712:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.748639:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44169 00002000:00080000:0.0:1783520935.748647:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1405 count 1 00000100:00100000:2.0:1783520935.748651:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515345c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+99:3628:x1870152993075840:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.748660:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc02533f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x57d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.748667:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1406 count 1 00010000:00010000:0.0:1783520935.748671:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003f4dad36/0xd5401cbabc01f5a6 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x57d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efddf expref: 901 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.748677:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x57d:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.748680:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x57d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.748682:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025346 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x57e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.748684:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc02533f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x57d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.748706:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c5500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993075456:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.748707:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000099248044/0xd5401cbabc01f72e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x57e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efde6 expref: 901 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.748712:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003f4dad36/0xd5401cbabc01f5a6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x57d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efddf expref: 901 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.748724:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025331 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x57b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.748730:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003f4dad36/0xd5401cbabc01f5a6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x57d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efddf expref: 901 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.748732:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x57e:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.748741:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x57e:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.748751:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025346 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x57e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.748752:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515345880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993076096:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.748774:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:2.0:1783520935.748780:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000099248044/0xd5401cbabc01f72e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x57e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efde6 expref: 901 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.748787:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e0f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993075968:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.748791:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470cc0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.748795:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000099248044/0xd5401cbabc01f72e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x57e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efde6 expref: 901 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.748808:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cc7f100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993076224:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.748814:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9c000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993075584:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.748824:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025338 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x57c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.748895:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.748909:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f180 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.748910:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.748923:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002374784 00000100:00100000:1.0:1783520935.748928:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002374784 00000100:00100000:2.0:1783520935.748932:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993075968 00000100:00100000:2.0:1783520935.748937:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993075968 00000100:00100000:0.0:1783520935.748942:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.748950:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ce9d266d/0xd5401cbabc01f59f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x57b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdd1 expref: 902 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.748961:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002374400 00000100:00100000:0.0:1783520935.748965:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002374400 00000100:00100000:2.0:1783520935.748971:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44170 00000100:00100000:1.0:1783520935.748979:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35756 00010000:00010000:0.0:1783520935.748980:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000099d089f6/0xd5401cbabc01f712 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x576:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdae expref: 903 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.748984:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350d165f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+100:3626:x1870152993075968:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.748992:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357ec50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+903:2389:x1870153002374784:12345-192.168.204.33@tcp:103:ldlm_bl.0 00002000:00080000:2.0:1783520935.749003:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1407 count 1 00010000:00010000:1.0:1783520935.749004:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.749010:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35757 00010000:00010000:1.0:1783520935.749012:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ce9d266d/0xd5401cbabc01f59f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x57b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdd1 expref: 903 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.749021:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02534d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x57f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.749021:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515346680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+903:2388:x1870153002374400:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.749033:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ce9d266d/0xd5401cbabc01f59f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x57b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdd1 expref: 903 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.749038:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.749044:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000099d089f6/0xd5401cbabc01f712 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x576:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdae expref: 903 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.749049:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b272cc8f/0xd5401cbabc01f616 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x57f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efded expref: 903 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.749063:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ce9d266d/0xd5401cbabc01f59f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x57b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdd1 expref: 903 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.749067:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000099d089f6/0xd5401cbabc01f712 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x576:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdae expref: 903 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.749071:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x57f:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.749081:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x57f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.749091:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02534d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x57f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.749094:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x57b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.749098:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000099d089f6/0xd5401cbabc01f712 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x576:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdae expref: 902 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.749104:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x57b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.749112:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x576:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.749113:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025331 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x57b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.749119:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x576:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.749120:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b272cc8f/0xd5401cbabc01f616 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x57f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efded expref: 901 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.749127:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025315 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x576:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.749132:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025331 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x57b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.749147:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025315 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x576:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.749150:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b272cc8f/0xd5401cbabc01f616 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x57f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efded expref: 901 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.749165:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cc7df80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993076352:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.749167:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.749167:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.749180:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025331 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x57b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.749192:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515346680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+901:2388:x1870153002374400:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 175us (250us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.749194:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025331 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x57b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.749201:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357ec50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+901:2389:x1870153002374784:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 210us (302us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.749203:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943515746cc0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.749203:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35757 00010000:00010000:3.0:1783520935.749218:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025331 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x57b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.749219:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35756 00010000:00010000:0.0:1783520935.749234:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025315 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x576:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.749236:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025331 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x57b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.749253:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025315 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x576:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.749264:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.749264:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025315 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x576:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.749275:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025315 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x576:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.749290:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520935.749301:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1783520935.749305:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025331 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x57b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.749312:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6800 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520935.749317:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cc7fb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993075328:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.749317:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025331 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x57b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.749332:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025331 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x57b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.749340:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02532a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x57a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.749343:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025331 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x57b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.749375:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025331 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x57b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.749396:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.749401:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025315 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x576:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.749409:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514f9b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+102:3625:x1870152993075072:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2195us (2332us total) trans 4294975664 rc 0/0 00010000:00010000:1.0:1783520935.749411:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5e00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520935.749412:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517245c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+102:3628:x1870152993073792:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3384us (3838us total) trans 4294975665 rc 0/0 00010000:00010000:1.0:1783520935.749418:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025315 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x576:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.749427:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44166 00010000:00010000:1.0:1783520935.749435:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025315 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x576:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.749441:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44162 00010000:00010000:1.0:1783520935.749454:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025315 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x576:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.749462:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025315 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x576:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.749465:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e1c5180/2, rc 0, transno 4294975664 00000100:00100000:3.0:1783520935.749482:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993075072:0@lo:6:osp-syn-1-0.0 00000004:00080000:0.0:1783520935.749483:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517244a80/2, rc 0, transno 4294975665 00010000:00010000:1.0:1783520935.749486:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.749491:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.749493:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517244a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993073792:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.749495:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.749518:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.749541:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.749548:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cc7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993076480:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.749577:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.749585:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.749601:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002374656 00000100:00100000:3.0:1783520935.749606:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002374656 00000100:00100000:0.0:1783520935.749612:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.749614:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993076480 00000100:00100000:2.0:1783520935.749620:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993076480 00000100:00100000:1.0:1783520935.749628:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cc7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993076608:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.749631:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000003387dfa/0xd5401cbabc022938 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x57a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3028 expref: 900 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.749657:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44171 00000100:00100000:1.0:1783520935.749658:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.749661:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002374528 00000100:00100000:3.0:1783520935.749665:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002374528 00000100:00100000:2.0:1783520935.749672:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+101:3626:x1870152993076480:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.749686:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1408 count 1 00010000:00010000:3.0:1783520935.749694:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000011832fe6/0xd5401cbabc01f598 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x579:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdc3 expref: 901 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.749695:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002374912 00010000:00010000:2.0:1783520935.749699:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025354 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x580:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.749702:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002374912 00000100:00100000:0.0:1783520935.749716:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993076608 00010000:00010000:2.0:1783520935.749721:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008422dba6/0xd5401cbabc01f8e7 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x580:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efdf4 expref: 902 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.749721:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993076608 00010000:00010000:1.0:1783520935.749723:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000020b97928/0xd5401cbabc01f727 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x57c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdd8 expref: 902 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.749734:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35758 00010000:00010000:2.0:1783520935.749742:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x580:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.749747:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x580:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.749749:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515344380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+902:2388:x1870153002374656:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.749752:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44172 00010000:00010000:2.0:1783520935.749753:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025354 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x580:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.749763:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35759 00010000:00010000:3.0:1783520935.749765:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.749768:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+102:3627:x1870152993076608:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.749774:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000003387dfa/0xd5401cbabc022938 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x57a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3028 expref: 902 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.749776:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515344000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+902:2388:x1870153002374528:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.749784:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008422dba6/0xd5401cbabc01f8e7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x580:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdf4 expref: 902 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.749785:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1409 count 1 00010000:00010000:1.0:1783520935.749790:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.749803:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000011832fe6/0xd5401cbabc01f598 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x579:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdc3 expref: 902 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.749803:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc02535b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x581:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.749813:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008422dba6/0xd5401cbabc01f8e7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x580:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdf4 expref: 902 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.749819:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000003387dfa/0xd5401cbabc022938 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x57a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3028 expref: 902 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.749828:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000403a1e6d/0xd5401cbabc01f5ad lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x581:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efdfb expref: 902 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.749839:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000011832fe6/0xd5401cbabc01f598 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x579:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdc3 expref: 902 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.749847:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cc7c700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993076736:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.749848:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x581:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.749852:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000003387dfa/0xd5401cbabc022938 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x57a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3028 expref: 902 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.749857:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x581:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.749865:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc02535b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x581:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.749870:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x57a:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.749873:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000011832fe6/0xd5401cbabc01f598 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x579:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdc3 expref: 901 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.749875:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x57a:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520935.749882:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02532a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x57a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.749885:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000403a1e6d/0xd5401cbabc01f5ad lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x581:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdfb expref: 901 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.749893:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f000 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.749909:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02532a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x57a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.749910:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000403a1e6d/0xd5401cbabc01f5ad lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x581:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdfb expref: 901 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.749916:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x579:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.749921:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cc7f100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993076224:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.749925:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x579:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.749937:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025323 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x579:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.749938:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515345500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993076864:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.749943:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025346 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x57e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.749951:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.749960:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025323 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x579:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.749964:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470b40 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.749979:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515344380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+900:2388:x1870153002374656:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 232us (435us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.749986:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35760 00010000:00010000:1.0:1783520935.749993:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.749995:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35758 00000100:00100000:0.0:1783520935.749999:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515347480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+900:2388:x1870153002374912:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.750010:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.750017:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000020b97928/0xd5401cbabc01f727 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x57c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdd8 expref: 899 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.750025:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515344000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+899:2388:x1870153002374528:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 249us (444us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.750034:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000020b97928/0xd5401cbabc01f727 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x57c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdd8 expref: 899 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.750037:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35759 00010000:00010000:0.0:1783520935.750056:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000020b97928/0xd5401cbabc01f727 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x57c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdd8 expref: 898 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.750067:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025323 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x579:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.750068:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x57c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.750072:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02532a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x57a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.750077:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x57c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.750087:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025338 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x57c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.750091:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025323 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x579:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.750096:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02532a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x57a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.750112:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025338 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x57c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.750114:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025323 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x579:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.750119:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02532a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x57a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.750137:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025323 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x579:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.750139:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02532a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x57a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.750148:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.750168:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520935.750175:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:0.0:1783520935.750175:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515347480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+897:2388:x1870153002374912:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 179us (565us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.750185:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35760 00000100:00100000:0.0:1783520935.750215:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515345880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993076096:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.750229:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc02533f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x57d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.750244:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+104:3627:x1870152993074688:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3550us (3646us total) trans 4294975666 rc 0/0 00010000:00010000:0.0:1783520935.750248:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025338 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x57c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.750254:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44164 00010000:00010000:0.0:1783520935.750262:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025338 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x57c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.750272:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025338 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x57c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.750274:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e0f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+103:3628:x1870152993074944:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3290us (3373us total) trans 4294975667 rc 0/0 00000004:00080000:1.0:1783520935.750292:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e1c5c00/2, rc 0, transno 4294975666 00000100:00100000:2.0:1783520935.750293:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44165 00010000:00010000:0.0:1783520935.750301:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025338 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x57c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.750311:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993074688:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.750318:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to regular blp list, 3 blwis in pool 00010000:00010000:3.0:1783520935.750320:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:2.0:1783520935.750321:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520935.750333:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025323 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x579:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.750333:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02532a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x57a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.750353:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.750355:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.750356:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.750385:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025338 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x57c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.750387:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025323 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x579:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.750388:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02532a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x57a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.750409:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.750415:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02532a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x57a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.750417:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025338 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x57c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.750420:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025323 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x579:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.750422:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514f99c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+103:3625:x1870152993075200:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3257us (3337us total) trans 4294975668 rc 0/0 00000100:00100000:0.0:1783520935.750442:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44167 00010000:00010000:1.0:1783520935.750444:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025338 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x57c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.750447:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02532a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x57a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.750452:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025323 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x579:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.750470:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025338 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x57c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.750471:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02532a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x57a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.750472:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025323 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x579:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.750494:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c1a73293/0xd5401cbabc025338 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x57c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.750514:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.750523:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.750527:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.750528:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.750531:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.750533:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.750535:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.750536:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.750539:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.750542:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.750551:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993076992:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.750582:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002375040 00000100:00100000:1.0:1783520935.750587:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002375040 00010000:00010000:1.0:1783520935.750604:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003f4dad36/0xd5401cbabc01f5a6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x57d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efddf expref: 897 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.750626:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35761 00000100:00100000:1.0:1783520935.750633:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515345880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+897:2389:x1870153002375040:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.750641:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.750645:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003f4dad36/0xd5401cbabc01f5a6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x57d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efddf expref: 897 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.750656:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003f4dad36/0xd5401cbabc01f5a6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x57d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efddf expref: 897 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.750668:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003f4dad36/0xd5401cbabc01f5a6 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x57d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efddf expref: 897 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.750679:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x57d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.750682:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x57d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.750686:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc02533f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x57d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.750695:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc02533f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x57d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.750709:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e0f7480/2, rc 0, transno 4294975668 00010000:00010000:1.0:1783520935.750718:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.750722:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e0f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993075200:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.750724:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.750728:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc02533f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x57d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.750738:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc02533f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x57d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.750747:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc02533f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x57d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.750752:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515345880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+896:2389:x1870153002375040:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 116us (249us total) trans 0 rc 0/0 00000004:00080000:2.0:1783520935.750761:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e0f7b80/2, rc 0, transno 4294975667 00010000:00010000:0.0:1783520935.750763:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc02533f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x57d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.750771:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e0f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993074944:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.750779:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35761 00010000:00010000:0.0:1783520935.750791:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520935.750810:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cc7df80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993076352:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.750828:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02534d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x57f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.750842:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.750856:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993077248:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.750887:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515344700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+103:3628:x1870152993075712:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2257us (2448us total) trans 4294975669 rc 0/0 00000100:00100000:3.0:1783520935.750888:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000004:00080000:2.0:1783520935.750897:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e0f5180/2, rc 0, transno 4294975669 00000100:00100000:0.0:1783520935.750902:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44168 00000100:00100000:1.0:1783520935.750905:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.750913:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e0f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993075712:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.750924:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993076992 00000100:00100000:0.0:1783520935.750929:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993076992 00000100:00100000:1.0:1783520935.750933:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1783520935.750939:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993077120:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.750955:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002375168 00000100:00100000:0.0:1783520935.750959:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993077248 00000100:00100000:0.0:1783520935.750963:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993077248 00000100:00100000:2.0:1783520935.750968:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1783520935.750968:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.750979:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cc7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993077376:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.750988:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993077120 00000100:00100000:0.0:1783520935.750992:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993077120 00000100:00100000:1.0:1783520935.751009:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002375296 00000100:00100000:2.0:1783520935.751010:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.751016:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002375296 00000100:00100000:0.0:1783520935.751017:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993077376 00000100:00100000:0.0:1783520935.751022:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993077376 00010000:00010000:1.0:1783520935.751039:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b272cc8f/0xd5401cbabc01f616 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x57f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efded expref: 896 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.751050:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44173 00000100:00100000:2.0:1783520935.751055:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cc7c700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993076736:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.751061:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cc7f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+106:3628:x1870152993076992:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.751067:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35762 00000100:00100000:3.0:1783520935.751072:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002375168 00010000:00010000:2.0:1783520935.751076:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025354 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x580:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.751079:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357c050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+897:2388:x1870153002375296:12345-192.168.204.33@tcp:103:ldlm_bl.0 00002000:00080000:0.0:1783520935.751083:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1410 count 1 00010000:00010000:3.0:1783520935.751092:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000099248044/0xd5401cbabc01f72e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x57e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efde6 expref: 897 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.751093:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.751099:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025362 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x582:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.751101:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b272cc8f/0xd5401cbabc01f616 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x57f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efded expref: 897 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.751108:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44174 00000100:00100000:2.0:1783520935.751122:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+106:3626:x1870152993077248:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.751126:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f7c0b404/0xd5401cbabc01f7a5 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x582:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efee2 expref: 897 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.751128:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35763 00010000:00010000:1.0:1783520935.751128:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b272cc8f/0xd5401cbabc01f616 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x57f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efded expref: 897 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.751140:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1412 count 1 00000100:00100000:3.0:1783520935.751142:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350d165180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+897:2390:x1870153002375168:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.751150:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x582:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.751155:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x582:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.751156:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.751157:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025369 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x584:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.751165:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b272cc8f/0xd5401cbabc01f616 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x57f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efded expref: 897 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.751165:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025362 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x582:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.751166:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000099248044/0xd5401cbabc01f72e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x57e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efde6 expref: 897 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.751173:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003babf63d/0xd5401cbabc01f7ac lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x584:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efef0 expref: 897 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.751194:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x57f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.751195:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x584:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.751198:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x57f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.751201:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000099248044/0xd5401cbabc01f72e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x57e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efde6 expref: 896 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.751201:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x584:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.751203:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02534d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x57f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.751206:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f7c0b404/0xd5401cbabc01f7a5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x582:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efee2 expref: 896 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.751208:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025369 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x584:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.751232:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02534d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x57f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.751237:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f7c0b404/0xd5401cbabc01f7a5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x582:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efee2 expref: 896 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.751241:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000099248044/0xd5401cbabc01f72e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x57e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efde6 expref: 896 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.751250:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003babf63d/0xd5401cbabc01f7ac lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x584:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efef0 expref: 896 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.751265:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.751266:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515346d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993077504:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.751274:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003babf63d/0xd5401cbabc01f7ac lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x584:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efef0 expref: 896 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.751287:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cc7c700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993077632:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.751295:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357c050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+896:2388:x1870153002375296:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 217us (410us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.751304:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35762 00000100:00100000:0.0:1783520935.751307:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470d80 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.751320:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f000 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.751320:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.751326:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x57e:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.751326:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc02533f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x57d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.751335:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44175 00010000:00010000:3.0:1783520935.751336:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x57e:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.751338:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02534d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x57f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.751344:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.751347:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025346 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x57e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.751352:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+106:3626:x1870152993077120:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.751352:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc02533f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x57d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.751353:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02534d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x57f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.751387:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc02533f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x57d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.751394:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025346 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x57e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.751403:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1411 count 1 00010000:00010000:0.0:1783520935.751408:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc02533f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x57d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.751417:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025370 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x583:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.751435:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02534d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x57f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.751435:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc02533f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x57d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.751437:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.751441:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000dfc5a096/0xd5401cbabc01fc3d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x583:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efee9 expref: 894 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.751461:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02534d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x57f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.751463:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350d165180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+894:2390:x1870153002375168:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 322us (621us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.751465:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x583:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.751468:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.751471:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.751475:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x583:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.751477:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.751482:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35763 00010000:00010000:2.0:1783520935.751482:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.751483:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025370 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x583:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.751512:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44176 00010000:00010000:1.0:1783520935.751520:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000dfc5a096/0xd5401cbabc01fc3d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x583:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efee9 expref: 893 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.751524:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cc7ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+106:3628:x1870152993077376:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.751542:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1413 count 1 00010000:00010000:1.0:1783520935.751546:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000dfc5a096/0xd5401cbabc01fc3d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x583:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efee9 expref: 893 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.751556:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025377 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x585:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.751561:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c6300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993077760:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.751582:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350d165f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+107:3626:x1870152993075968:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2598us (2668us total) trans 4294975670 rc 0/0 00010000:00010000:0.0:1783520935.751582:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000fade76a5/0xd5401cbabc01fc44 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x585:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efef7 expref: 893 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.751588:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e0f5f80/2, rc 0, transno 4294975670 00010000:00010000:0.0:1783520935.751599:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x585:0x0].0x0 since=37s v=(0 0) 00000100:00100000:3.0:1783520935.751601:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e0f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993075968:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.751601:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44170 00010000:00010000:0.0:1783520935.751605:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x585:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.751607:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6cc0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.751613:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025377 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x585:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.751627:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.751633:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02534d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x57f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.751634:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000fade76a5/0xd5401cbabc01fc44 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x585:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efef7 expref: 893 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.751654:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.751659:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000fade76a5/0xd5401cbabc01fc44 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x585:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efef7 expref: 893 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.751660:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02534d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x57f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.751683:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02534d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x57f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.751687:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515346680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993077888:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.751703:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02534d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x57f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.751709:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02534d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x57f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.751718:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470900 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.751719:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.751721:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.751724:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.751764:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025346 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x57e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.751765:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cc7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993078016:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.751776:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025346 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x57e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.751782:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025346 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x57e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.751787:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025346 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x57e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.751795:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.751798:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520935.751801:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.751805:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.751810:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025346 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x57e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.751817:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.751822:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025346 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x57e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.751832:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025346 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x57e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.751845:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993078016 00010000:00010000:1.0:1783520935.751846:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025346 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x57e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.751850:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993078016 00010000:00010000:1.0:1783520935.751858:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025346 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x57e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.751866:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515345c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+108:3628:x1870152993075840:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3220us (3367us total) trans 4294975671 rc 0/0 00000100:00100000:2.0:1783520935.751873:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44169 00010000:00010000:1.0:1783520935.751879:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.751883:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.751886:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44177 00010000:00010000:1.0:1783520935.751888:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.751890:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.751899:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350d165180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+107:3628:x1870152993078016:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.751906:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002375424 00000100:00100000:1.0:1783520935.751911:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002375424 00002000:00080000:3.0:1783520935.751915:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1414 count 1 00000004:00080000:2.0:1783520935.751915:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e0f4a80/2, rc 0, transno 4294975671 00010000:00010000:1.0:1783520935.751926:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008422dba6/0xd5401cbabc01f8e7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x580:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdf4 expref: 894 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.751927:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e0f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993075840:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.751950:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002375552 00000100:00100000:1.0:1783520935.751952:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002375552 00010000:00010000:1.0:1783520935.751962:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000403a1e6d/0xd5401cbabc01f5ad lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x581:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdfb expref: 895 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.751981:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02537e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x586:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.751985:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35764 00000100:00100000:2.0:1783520935.751994:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35765 00010000:00010000:3.0:1783520935.751998:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000506e65d1/0xd5401cbabc01f7b3 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x586:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859efefe expref: 895 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.752000:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515344380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+895:2389:x1870153002375424:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.752009:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515344000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+895:2389:x1870153002375552:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.752012:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x586:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.752016:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x586:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.752017:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.752019:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02537e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x586:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.752019:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.752025:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008422dba6/0xd5401cbabc01f8e7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x580:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdf4 expref: 895 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.752026:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000403a1e6d/0xd5401cbabc01f5ad lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x581:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdfb expref: 895 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.752026:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515345500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993076864:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.752045:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc02535b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x581:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.752050:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000403a1e6d/0xd5401cbabc01f5ad lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x581:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdfb expref: 895 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.752056:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008422dba6/0xd5401cbabc01f8e7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x580:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdf4 expref: 895 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.752081:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000403a1e6d/0xd5401cbabc01f5ad lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x581:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdfb expref: 895 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.752092:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008422dba6/0xd5401cbabc01f8e7 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x580:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efdf4 expref: 894 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.752093:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x581:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.752095:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000506e65d1/0xd5401cbabc01f7b3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x586:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efefe expref: 894 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.752103:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x581:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.752113:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc02535b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x581:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.752119:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000506e65d1/0xd5401cbabc01f7b3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x586:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efefe expref: 893 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.752122:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x580:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.752131:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc02535b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x581:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.752131:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x580:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.752136:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350d167100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993078272:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.752141:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025354 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x580:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.752160:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025354 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x580:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.752166:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.752169:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872b40 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.752182:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.752182:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc02535b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x581:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.752194:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515344000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+893:2389:x1870153002375552:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 187us (308us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.752202:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc02535b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x581:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.752209:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35765 00000100:00100000:1.0:1783520935.752213:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515344380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+893:2389:x1870153002375424:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 214us (416us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.752220:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc02535b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x581:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.752226:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35764 00010000:00010000:0.0:1783520935.752234:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc02535b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x581:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.752253:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.752258:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025354 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x580:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.752264:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:2.0:1783520935.752265:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.752267:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025354 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x580:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.752273:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc02535b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x581:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.752277:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993078144:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.752278:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025354 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x580:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.752292:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.752297:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025354 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x580:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.752298:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc02535b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x581:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.752321:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc02535b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x581:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.752325:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.752337:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc02535b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x581:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.752345:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc02535b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x581:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.752355:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.752401:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.752406:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025354 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x580:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.752413:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.752417:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025354 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x580:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.752425:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025354 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x580:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.752429:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+109:3626:x1870152993076480:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2761us (2844us total) trans 4294975673 rc 0/0 00000100:00100000:0.0:1783520935.752440:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+109:3627:x1870152993076608:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2673us (2782us total) trans 4294975672 rc 0/0 00000100:00100000:1.0:1783520935.752443:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44171 00000004:00080000:3.0:1783520935.752450:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cc7ca80/2, rc 0, transno 4294975672 00000100:00100000:1.0:1783520935.752458:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993078144 00000100:00100000:0.0:1783520935.752459:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44172 00000100:00100000:1.0:1783520935.752462:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993078144 00000100:00100000:3.0:1783520935.752466:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cc7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993076608:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.752466:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025354 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x580:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.752479:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025354 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x580:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.752487:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44178 00010000:00010000:2.0:1783520935.752511:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.752513:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350d167800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+108:3626:x1870152993078144:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.752514:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.752519:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00002000:00080000:0.0:1783520935.752529:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1415 count 1 00010000:00010000:0.0:1783520935.752543:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025385 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x587:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.752555:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005ce9adc9/0xd5401cbabc01fc4b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x587:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859eff05 expref: 891 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.752563:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x587:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.752567:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x587:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.752570:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350d166300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993078400:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.752573:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025385 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x587:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.752590:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.752600:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005ce9adc9/0xd5401cbabc01fc4b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x587:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff05 expref: 891 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.752612:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cc7fb80/2, rc 0, transno 4294975673 00010000:00010000:0.0:1783520935.752612:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005ce9adc9/0xd5401cbabc01fc4b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x587:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff05 expref: 891 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.752616:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993078400 00000100:00100000:3.0:1783520935.752621:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cc7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993076480:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.752622:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993078400 00000100:00100000:0.0:1783520935.752636:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515345500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993078528:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.752661:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44179 00000100:00100000:3.0:1783520935.752664:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993078656:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.752676:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350d165880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+109:3626:x1870152993078400:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.752690:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00080000:1.0:1783520935.752690:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1416 count 1 00000100:00100000:0.0:1783520935.752694:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470780 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.752706:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02538c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x588:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.752713:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.752717:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e59c2202/0xd5401cbabc01f7ba lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x588:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859eff0c expref: 891 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.752720:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993078656 00010000:00010000:1.0:1783520935.752723:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x588:0x0].0x0 since=37s v=(0 0) 00000100:00100000:3.0:1783520935.752724:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993078656 00010000:00010000:1.0:1783520935.752727:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x588:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.752731:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02538c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x588:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.752747:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e59c2202/0xd5401cbabc01f7ba lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x588:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff0c expref: 891 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.752754:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44180 00010000:00010000:1.0:1783520935.752755:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e59c2202/0xd5401cbabc01f7ba lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x588:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff0c expref: 891 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.752768:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514f9ad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993078784:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.752771:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350d166680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+110:3627:x1870152993078656:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520935.752786:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1417 count 1 00000100:00100000:1.0:1783520935.752794:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6840 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.752799:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025393 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x589:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.752813:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000098009358/0xd5401cbabc01fc52 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x589:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859eff13 expref: 891 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.752821:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x589:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.752825:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x589:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.752829:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025393 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x589:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.752914:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000098009358/0xd5401cbabc01fc52 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x589:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff13 expref: 891 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.752930:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000098009358/0xd5401cbabc01fc52 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x589:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff13 expref: 891 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.752937:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350d166a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993078912:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.752994:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872f00 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.753118:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515346d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993077504:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.753130:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cc7c700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993077632:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.753140:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025362 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x582:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.753144:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c6300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993077760:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.753150:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025369 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x584:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.753165:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025370 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x583:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.753199:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350d167100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993078272:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.753204:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515346680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993077888:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.753218:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02537e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x586:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.753221:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025377 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x585:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.753570:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.753593:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.753596:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002375680 00000100:00100000:1.0:1783520935.753601:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002375680 00000100:00100000:2.0:1783520935.753621:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002375936 00000100:00100000:0.0:1783520935.753621:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.753626:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f7c0b404/0xd5401cbabc01f7a5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x582:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efee2 expref: 892 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.753628:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002375936 00010000:00010000:2.0:1783520935.753649:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000dfc5a096/0xd5401cbabc01fc3d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x583:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efee9 expref: 893 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.753657:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002376192 00000100:00100000:0.0:1783520935.753657:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.753663:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35766 00000100:00100000:3.0:1783520935.753664:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002376192 00000100:00100000:1.0:1783520935.753679:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515346680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+894:2387:x1870153002375680:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.753685:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002375808 00010000:00010000:3.0:1783520935.753690:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000506e65d1/0xd5401cbabc01f7b3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x586:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efefe expref: 894 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.753690:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002375808 00010000:00010000:1.0:1783520935.753690:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.753701:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.753703:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f7c0b404/0xd5401cbabc01f7a5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x582:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efee2 expref: 895 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.753709:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003babf63d/0xd5401cbabc01f7ac lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x584:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efef0 expref: 895 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.753720:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002376064 00000100:00100000:3.0:1783520935.753725:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002376064 00010000:00010000:1.0:1783520935.753725:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f7c0b404/0xd5401cbabc01f7a5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x582:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efee2 expref: 895 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.753734:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35767 00010000:00010000:3.0:1783520935.753742:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000fade76a5/0xd5401cbabc01fc44 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x585:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efef7 expref: 896 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.753743:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35768 00000100:00100000:0.0:1783520935.753749:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515347480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+896:2387:x1870153002375936:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.753750:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f7c0b404/0xd5401cbabc01f7a5 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x582:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efee2 expref: 896 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.753761:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515345880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+896:2387:x1870153002376192:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.753767:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520935.753775:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35769 00010000:00010000:2.0:1783520935.753777:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.753778:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x582:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.753778:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000dfc5a096/0xd5401cbabc01fc3d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x583:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efee9 expref: 895 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.753787:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000506e65d1/0xd5401cbabc01f7b3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x586:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efefe expref: 895 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.753788:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x582:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.753789:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515346d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+895:2388:x1870153002375808:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.753801:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025362 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x582:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.753805:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.753815:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003babf63d/0xd5401cbabc01f7ac lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x584:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efef0 expref: 895 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.753819:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000506e65d1/0xd5401cbabc01f7b3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x586:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efefe expref: 895 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.753830:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025362 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x582:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.753831:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000dfc5a096/0xd5401cbabc01fc3d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x583:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efee9 expref: 895 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.753848:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003babf63d/0xd5401cbabc01f7ac lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x584:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efef0 expref: 895 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.753858:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000506e65d1/0xd5401cbabc01f7b3 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x586:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efefe expref: 895 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.753866:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000dfc5a096/0xd5401cbabc01fc3d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x583:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efee9 expref: 895 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.753868:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.753873:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003babf63d/0xd5401cbabc01f7ac lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x584:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efef0 expref: 894 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.753874:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x586:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.753884:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x586:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.753889:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x583:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.753891:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02537e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x586:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.753892:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x584:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.753893:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515346680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+892:2387:x1870153002375680:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 218us (322us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.753899:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x583:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.753900:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x584:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.753911:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35766 00010000:00010000:3.0:1783520935.753914:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025369 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x584:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.753914:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025370 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x583:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.753918:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02537e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x586:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.753928:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35770 00010000:00010000:3.0:1783520935.753938:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025369 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x584:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.753939:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357cc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+891:2388:x1870153002376064:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.753942:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025370 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x583:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.753952:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.753957:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.753961:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000fade76a5/0xd5401cbabc01fc44 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x585:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efef7 expref: 891 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.753970:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000fade76a5/0xd5401cbabc01fc44 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x585:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efef7 expref: 891 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.753971:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.753982:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.753987:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000fade76a5/0xd5401cbabc01fc44 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x585:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859efef7 expref: 891 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.753988:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515345880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+891:2387:x1870153002376192:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 229us (364us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.753995:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515346d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+891:2388:x1870153002375808:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 210us (342us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.754007:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35768 00010000:00010000:1.0:1783520935.754008:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x585:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.754012:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515347480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+890:2387:x1870153002375936:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 266us (415us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.754014:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x585:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.754018:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35769 00010000:00010000:1.0:1783520935.754024:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025377 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x585:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.754031:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35767 00010000:00010000:1.0:1783520935.754039:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025377 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x585:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.754046:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025370 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x583:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.754056:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025370 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x583:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.754056:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.754067:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025377 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x585:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.754068:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025370 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x583:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.754079:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357cc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+887:2388:x1870153002376064:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 140us (382us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.754091:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025377 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x585:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.754094:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025370 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x583:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.754095:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35770 00010000:00010000:0.0:1783520935.754115:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025377 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x585:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.754122:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025369 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x584:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.754125:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.754126:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025377 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x585:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.754137:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025369 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x584:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.754147:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025369 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x584:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.754152:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025369 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x584:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.754152:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520935.754157:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1783520935.754165:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025370 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x583:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.754170:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520935.754176:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.754181:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025370 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x583:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.754190:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025370 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x583:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.754205:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025370 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x583:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.754212:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025370 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x583:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.754228:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+113:3626:x1870152993077120:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2881us (3260us total) trans 4294975674 rc 0/0 00010000:00010000:1.0:1783520935.754232:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00000100:00100000:3.0:1783520935.754238:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44175 00010000:00010000:1.0:1783520935.754239:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025377 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x585:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.754246:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+113:3626:x1870152993077248:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3126us (3346us total) trans 4294975675 rc 0/0 00010000:00010000:1.0:1783520935.754254:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfac00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520935.754257:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44174 00010000:00010000:1.0:1783520935.754261:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025377 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x585:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.754271:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025377 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x585:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.754276:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e1c4000/2, rc 0, transno 4294975674 00010000:00010000:2.0:1783520935.754278:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520935.754280:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cc7ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+111:3628:x1870152993077376:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2758us (3268us total) trans 4294975676 rc 0/0 00010000:00010000:2.0:1783520935.754285:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025369 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x584:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.754289:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025377 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x585:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.754292:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993077120:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.754293:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44176 00010000:00010000:2.0:1783520935.754300:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.754306:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025369 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x584:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.754307:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025377 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x585:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.754309:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e1c4380/2, rc 0, transno 4294975675 00000100:00100000:3.0:1783520935.754317:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993077248:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.754331:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025369 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x584:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.754336:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.754339:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.754342:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.754347:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025369 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x584:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.754353:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025362 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x582:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.754420:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025369 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x584:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.754433:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.754435:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.754437:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.754438:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025362 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x582:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.754441:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.754451:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025362 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x582:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.754452:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350d164700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993079040:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.754469:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025362 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x582:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.754478:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.754483:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000004:00080000:2.0:1783520935.754520:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cc7df80/2, rc 0, transno 4294975676 00000100:00100000:2.0:1783520935.754526:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cc7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993077376:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.754534:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.754541:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025362 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x582:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.754550:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfa200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.754554:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025362 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x582:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.754564:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025362 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x582:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.754572:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cc7f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+111:3628:x1870152993076992:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3511us (3847us total) trans 4294975677 rc 0/0 00000100:00100000:1.0:1783520935.754583:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44173 00000004:00080000:2.0:1783520935.754584:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e1c7b80/2, rc 0, transno 4294975677 00010000:00010000:0.0:1783520935.754584:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025362 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x582:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.754596:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025362 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x582:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.754598:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993076992:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.754599:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993079040 00000100:00100000:1.0:1783520935.754606:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993079040 00000100:00100000:2.0:1783520935.754619:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350d165c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993079168:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.754624:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.754629:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.754633:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.754649:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44181 00000100:00100000:2.0:1783520935.754655:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.754660:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635602450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+111:3628:x1870152993079040:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.754675:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1418 count 1 00000100:00100000:0.0:1783520935.754679:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515345500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993078528:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.754691:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc02539a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x58a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.754692:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:0.0:1783520935.754699:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025385 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x587:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.754704:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350d164a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993079296:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.754706:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000049642385/0xd5401cbabc022970 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x58a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3060 expref: 886 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.754723:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.754724:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x58a:0x0].0x0 since=36s v=(0 0) 00010000:00010000:1.0:1783520935.754728:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x58a:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.754731:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc02539a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x58a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.754741:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.754750:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002376448 00010000:00010000:1.0:1783520935.754751:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000049642385/0xd5401cbabc022970 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x58a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3060 expref: 886 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.754756:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002376448 00000100:00100000:3.0:1783520935.754758:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.754776:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000049642385/0xd5401cbabc022970 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x58a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3060 expref: 887 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.754777:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993079168 00010000:00010000:0.0:1783520935.754779:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005ce9adc9/0xd5401cbabc01fc4b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x587:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff05 expref: 887 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.754783:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993079168 00000100:00100000:3.0:1783520935.754789:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.754808:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad3b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993079424:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.754814:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993079296 00000100:00100000:2.0:1783520935.754818:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993079296 00000100:00100000:0.0:1783520935.754820:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002376576 00000100:00100000:0.0:1783520935.754824:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002376576 00010000:00010000:0.0:1783520935.754842:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000098009358/0xd5401cbabc01fc52 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x589:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff13 expref: 888 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.754844:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993079552:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.754849:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44182 00000100:00100000:1.0:1783520935.754857:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6540 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.754863:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cba1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+113:3625:x1870152993079168:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.754871:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002376320 00000100:00100000:0.0:1783520935.754876:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002376320 00002000:00080000:2.0:1783520935.754878:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1419 count 1 00000100:00100000:1.0:1783520935.754883:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35771 00000100:00100000:3.0:1783520935.754886:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.754892:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e59c2202/0xd5401cbabc01f7ba lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x588:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff0c expref: 889 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.754896:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0253a1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x58b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.754899:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350d166d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+889:2390:x1870153002376448:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.754914:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.754924:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005ce9adc9/0xd5401cbabc01fc4b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x587:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff05 expref: 889 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.754928:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35772 00010000:00010000:2.0:1783520935.754928:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000bcb61d14/0xd5401cbabc01fc59 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x58b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859eff21 expref: 889 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.754939:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35773 00000100:00100000:3.0:1783520935.754941:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350d167100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+889:2387:x1870153002376576:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.754954:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x58b:0x0].0x0 since=37s v=(0 0) 00000100:00100000:0.0:1783520935.754954:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c6300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+889:2387:x1870153002376320:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.754955:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.754957:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005ce9adc9/0xd5401cbabc01fc4b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x587:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff05 expref: 889 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.754964:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000098009358/0xd5401cbabc01fc52 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x589:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff13 expref: 889 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.754964:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x58b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.754971:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.754975:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0253a1 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x58b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.754982:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e59c2202/0xd5401cbabc01f7ba lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x588:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff0c expref: 889 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.754989:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000098009358/0xd5401cbabc01fc52 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x589:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff13 expref: 889 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.754990:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005ce9adc9/0xd5401cbabc01fc4b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x587:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff05 expref: 889 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.755016:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000bcb61d14/0xd5401cbabc01fc59 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x58b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff21 expref: 888 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.755019:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x587:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.755022:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x589:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.755027:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e59c2202/0xd5401cbabc01f7ba lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x588:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff0c expref: 888 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.755029:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x587:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.755032:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x589:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.755041:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025385 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x587:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.755042:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025393 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x589:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.755047:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000bcb61d14/0xd5401cbabc01fc59 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x58b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff21 expref: 888 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.755055:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x588:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.755065:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x588:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.755066:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025393 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x589:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.755069:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025385 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x587:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.755077:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02538c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x588:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.755079:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94361103b100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993079680:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.755094:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.755097:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02538c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x588:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.755108:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.755117:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f000 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.755117:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.755124:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350d167100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+888:2387:x1870153002376576:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 185us (368us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.755134:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35772 00000100:00100000:1.0:1783520935.755138:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350d166d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+888:2390:x1870153002376448:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 241us (416us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.755141:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993079552 00000100:00100000:2.0:1783520935.755146:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993079552 00000100:00100000:0.0:1783520935.755148:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c6300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+887:2387:x1870153002376320:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 195us (360us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.755153:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35771 00000100:00100000:0.0:1783520935.755163:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35773 00000100:00100000:3.0:1783520935.755178:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44183 00000100:00100000:2.0:1783520935.755185:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44184 00000100:00100000:3.0:1783520935.755188:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cc7c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+114:3625:x1870152993079296:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.755197:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435079f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+114:3627:x1870152993079552:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520935.755200:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1420 count 1 00010000:00010000:0.0:1783520935.755202:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025385 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x587:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.755204:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e59c2202/0xd5401cbabc01f7ba lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x588:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff0c expref: 885 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.755214:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1421 count 1 00010000:00010000:3.0:1783520935.755215:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0253a8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x58c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.755227:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025385 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x587:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.755228:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0253af lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x58d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.755235:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514f9ad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993078784:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.755245:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000060f5294a/0xd5401cbabc01f7c1 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x58c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859eff28 expref: 884 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.755254:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025385 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x587:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.755258:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02538c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x588:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.755260:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008b4c36ed/0xd5401cbabc01fc60 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x58d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859eff2f expref: 884 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.755261:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x58c:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.755271:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x58c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.755280:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025385 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x587:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.755281:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0253a8 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x58c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.755287:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02538c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x588:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.755288:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x58d:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.755297:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x58d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.755308:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0253af lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x58d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.755314:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02538c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x588:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.755316:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.755334:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02538c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x588:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.755342:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000060f5294a/0xd5401cbabc01f7c1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x58c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff28 expref: 884 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.755344:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02538c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x588:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.755347:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008b4c36ed/0xd5401cbabc01fc60 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x58d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff2f expref: 884 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.755378:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000060f5294a/0xd5401cbabc01f7c1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x58c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff28 expref: 884 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.755399:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:3.0:1783520935.755401:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f0380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993079808:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.755405:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008b4c36ed/0xd5401cbabc01fc60 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x58d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff2f expref: 884 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.755418:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94361103b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993079936:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.755426:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520935.755430:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.755432:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025385 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x587:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.755437:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02538c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x588:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.755441:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872b40 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.755443:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfa000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.755450:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025385 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x587:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.755451:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.755458:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02538c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x588:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.755462:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350d167800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+115:3626:x1870152993078144:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2964us (3107us total) trans 4294975678 rc 0/0 00010000:00010000:1.0:1783520935.755463:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025385 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x587:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.755472:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cba1880/2, rc 0, transno 4294975678 00010000:00010000:2.0:1783520935.755475:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02538c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x588:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.755481:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44178 00000100:00100000:3.0:1783520935.755486:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993078144:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.755495:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025385 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x587:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.755517:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02538c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x588:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.755518:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025385 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x587:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.755540:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02538c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x588:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.755548:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:1.0:1783520935.755554:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.755558:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.755560:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993080064:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.755563:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.755565:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.755569:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.755572:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.755594:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.755614:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f180 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.755647:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000098009358/0xd5401cbabc01fc52 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x589:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff13 expref: 884 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.755664:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350d166a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993078912:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.755670:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350d165880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+115:3626:x1870152993078400:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2998us (3077us total) trans 4294975679 rc 0/0 00000100:00100000:0.0:1783520935.755675:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993080064 00000100:00100000:1.0:1783520935.755680:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44179 00000004:00080000:2.0:1783520935.755681:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350d166300/2, rc 0, transno 4294975679 00000100:00100000:0.0:1783520935.755682:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993080064 00000100:00100000:2.0:1783520935.755695:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350d166300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993078400:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.755714:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02537e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x586:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.755716:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44185 00010000:00010000:1.0:1783520935.755729:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02537e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x586:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.755729:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435079f3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+115:3627:x1870152993080064:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.755735:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00002000:00080000:0.0:1783520935.755745:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1422 count 1 00000100:00100000:2.0:1783520935.755746:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943611039f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993080192:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.755747:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02537e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x586:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.755754:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025393 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x589:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.755762:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0253b6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x58e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.755769:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02537e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x586:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.755778:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025393 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x589:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.755781:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.755793:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000072f46a36/0xd5401cbabc01f7c8 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x58e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859eff36 expref: 883 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.755803:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.755804:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x58e:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.755810:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.755810:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x58e:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.755815:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0253b6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x58e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.755817:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02537e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x586:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.755836:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.755842:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02537e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x586:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.755844:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025393 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x589:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.755850:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000072f46a36/0xd5401cbabc01f7c8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x58e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff36 expref: 883 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.755862:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02537e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x586:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.755870:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025393 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x589:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.755877:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000072f46a36/0xd5401cbabc01f7c8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x58e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff36 expref: 883 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.755887:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350d165180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+116:3628:x1870152993078016:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3989us (4085us total) trans 4294975680 rc 0/0 00010000:00010000:2.0:1783520935.755891:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02537e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x586:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.755898:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025393 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x589:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.755908:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44177 00000100:00100000:1.0:1783520935.755926:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993080192 00000100:00100000:1.0:1783520935.755930:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993080192 00000100:00100000:0.0:1783520935.755932:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515345500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993080320:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.755935:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02537e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x586:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.755957:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.755957:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44186 00010000:00010000:2.0:1783520935.755962:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.755966:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.755970:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943611038700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+116:3626:x1870152993080192:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.755970:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470780 going to sleep for 11 seconds 00002000:00080000:1.0:1783520935.755986:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1423 count 1 00010000:00010000:1.0:1783520935.755999:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0253bd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x58f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.756003:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.756013:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.756016:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000831ccd3b/0xd5401cbabc01fc67 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x58f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859eff3d expref: 883 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.756018:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025393 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x589:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.756039:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.756040:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x58f:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.756044:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025393 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x589:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.756046:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x58f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.756057:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0253bd lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x58f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.756060:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025393 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x589:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.756083:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025393 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x589:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.756092:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025393 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x589:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520935.756098:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cba0a80/2, rc 0, transno 4294975681 00010000:00010000:1.0:1783520935.756100:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000831ccd3b/0xd5401cbabc01fc67 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x58f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff3d expref: 883 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.756107:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.756108:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993078656:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.756110:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.756115:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.756119:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000831ccd3b/0xd5401cbabc01fc67 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x58f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff3d expref: 883 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.756129:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350d166680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+117:3627:x1870152993078656:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3361us (3437us total) trans 4294975681 rc 0/0 00000100:00100000:1.0:1783520935.756136:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad2d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993080448:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.756142:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44180 00000100:00100000:0.0:1783520935.756165:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94361103ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993080576:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.756176:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6f00 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.756195:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:1.0:1783520935.756208:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cc7ce00/2, rc 0, transno 4294975680 00000100:00100000:1.0:1783520935.756222:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cc7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993078016:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.756224:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993080576 00000100:00100000:2.0:1783520935.756230:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993080576 00000100:00100000:2.0:1783520935.756259:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44187 00000100:00100000:2.0:1783520935.756267:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515344000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+117:3626:x1870152993080576:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.756279:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1424 count 1 00010000:00010000:2.0:1783520935.756290:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0253c4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x590:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.756304:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000011d75cc7/0xd5401cbabc01f7cf lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x590:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859eff44 expref: 883 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.756311:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x590:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.756314:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x590:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.756318:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0253c4 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x590:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.756337:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000011d75cc7/0xd5401cbabc01f7cf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x590:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff44 expref: 883 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.756347:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000011d75cc7/0xd5401cbabc01f7cf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x590:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff44 expref: 883 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.756399:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94361103aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993080704:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.756438:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943515746cc0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.756511:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993080832:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.756536:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.756560:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993080832 00000100:00100000:0.0:1783520935.756566:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993080832 00000100:00100000:0.0:1783520935.756590:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44188 00000100:00100000:0.0:1783520935.756595:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943611038e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+118:3627:x1870152993080832:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.756604:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1425 count 1 00010000:00010000:0.0:1783520935.756613:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0253cb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x591:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.756625:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e8505347/0xd5401cbabc01fc6e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x591:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859eff4b expref: 883 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.756632:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x591:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.756635:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x591:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.756639:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0253cb lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x591:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.756658:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e8505347/0xd5401cbabc01fc6e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x591:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff4b expref: 883 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.756667:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e8505347/0xd5401cbabc01fc6e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x591:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff4b expref: 883 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.756674:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515344380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993080960:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.756700:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470900 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.756895:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94361103b100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993079680:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.756911:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f0380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993079808:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.756913:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0253a1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x58b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.756927:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0253a8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x58c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.756970:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94361103b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993079936:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.756984:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0253af lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x58d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.757006:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad3b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993079424:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.757021:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc02539a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x58a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.757546:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.757562:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.757577:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.757584:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002376704 00000100:00100000:2.0:1783520935.757590:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002376704 00000100:00100000:1.0:1783520935.757613:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002377088 00010000:00010000:2.0:1783520935.757616:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000bcb61d14/0xd5401cbabc01fc59 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x58b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff21 expref: 884 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.757619:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002377088 00000100:00100000:0.0:1783520935.757635:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.757641:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008b4c36ed/0xd5401cbabc01fc60 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x58d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff2f expref: 885 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.757648:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002376832 00000100:00100000:2.0:1783520935.757653:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002376832 00000100:00100000:1.0:1783520935.757671:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002377216 00000100:00100000:0.0:1783520935.757671:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515345500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993080320:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.757672:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000049642385/0xd5401cbabc022970 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x58a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3060 expref: 886 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.757678:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002377216 00010000:00010000:0.0:1783520935.757691:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0253b6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x58e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.757698:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000072f46a36/0xd5401cbabc01f7c8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x58e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff36 expref: 887 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.757705:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35774 00000100:00100000:2.0:1783520935.757719:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515346680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+887:2388:x1870153002376704:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.757722:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35775 00010000:00010000:2.0:1783520935.757732:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.757737:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515345880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+887:2388:x1870153002377088:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.757740:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000bcb61d14/0xd5401cbabc01fc59 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x58b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff21 expref: 887 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.757748:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35776 00010000:00010000:0.0:1783520935.757750:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.757758:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000bcb61d14/0xd5401cbabc01fc59 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x58b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff21 expref: 887 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.757760:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008b4c36ed/0xd5401cbabc01fc60 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x58d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff2f expref: 887 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.757764:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515346d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+887:2388:x1870153002376832:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.757781:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.757784:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008b4c36ed/0xd5401cbabc01fc60 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x58d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff2f expref: 887 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.757787:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000bcb61d14/0xd5401cbabc01fc59 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x58b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff21 expref: 887 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.757789:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000049642385/0xd5401cbabc022970 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x58a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3060 expref: 887 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.757809:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35777 00010000:00010000:2.0:1783520935.757818:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x58b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.757818:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008b4c36ed/0xd5401cbabc01fc60 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x58d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff2f expref: 886 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.757822:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000049642385/0xd5401cbabc022970 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x58a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3060 expref: 886 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.757824:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357e850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+886:2387:x1870153002377216:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.757828:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x58b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.757840:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.757841:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0253a1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x58b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.757848:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x58d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.757851:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000072f46a36/0xd5401cbabc01f7c8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x58e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff36 expref: 885 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.757857:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x58d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.757858:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000049642385/0xd5401cbabc022970 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x58a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3060 expref: 885 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.757862:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0253a1 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x58b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.757869:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0253af lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x58d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.757881:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000072f46a36/0xd5401cbabc01f7c8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x58e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff36 expref: 885 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.757889:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x58a:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.757897:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0253af lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x58d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.757899:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x58a:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.757902:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.757910:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc02539a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x58a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.757915:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000072f46a36/0xd5401cbabc01f7c8 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x58e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff36 expref: 884 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.757923:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.757926:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc02539a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x58a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.757940:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515346680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+884:2388:x1870153002376704:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 223us (390us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.757948:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515345880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+884:2388:x1870153002377088:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 212us (380us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.757956:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x58e:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.757962:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35774 00010000:00010000:3.0:1783520935.757963:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x58e:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.757967:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.757970:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0253b6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x58e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.757972:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35775 00010000:00010000:3.0:1783520935.757984:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0253b6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x58e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.757990:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0253a1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x58b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.757995:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515346d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+881:2388:x1870153002376832:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 236us (416us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.758008:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.758009:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0253a1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x58b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.758009:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35776 00000100:00100000:0.0:1783520935.758019:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94361103aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993080704:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.758025:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0253a1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x58b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.758035:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0253a1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x58b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.758037:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0253c4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x590:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.758038:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc02539a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x58a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.758046:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357e850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+880:2387:x1870153002377216:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 222us (406us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.758061:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.758064:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc02539a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x58a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.758067:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35777 00010000:00010000:1.0:1783520935.758078:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc02539a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x58a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.758092:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc02539a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x58a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.758110:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520935.758119:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0253b6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x58e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.758122:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520935.758131:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0253a1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x58b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.758133:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0253b6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x58e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.758154:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.758158:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0253b6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x58e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.758160:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0253a1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x58b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.758171:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cba1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+120:3625:x1870152993079168:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3308us (3515us total) trans 4294975682 rc 0/0 00010000:00010000:3.0:1783520935.758179:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0253b6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x58e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.758187:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635602450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+120:3628:x1870152993079040:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3527us (3706us total) trans 4294975683 rc 0/0 00010000:00010000:0.0:1783520935.758187:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0253a1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x58b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.758190:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44182 00000100:00100000:1.0:1783520935.758205:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44181 00010000:00010000:2.0:1783520935.758216:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520935.758217:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0253a1 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x58b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.758223:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc02539a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x58a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.758231:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.758238:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc02539a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x58a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.758260:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0253a1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x58b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.758264:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.758272:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc02539a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x58a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.758276:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.758282:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0253b6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x58e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.758288:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfbe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.758290:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0253b6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x58e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.758296:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0253b6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x58e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.758303:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0253b6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x58e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.758307:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0253b6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x58e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.758314:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.758324:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.758328:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.758330:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520935.758338:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350d164700/2, rc 0, transno 4294975683 00000100:00100000:1.0:1783520935.758354:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350d164700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993079040:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.758355:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc02539a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x58a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.758420:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc02539a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x58a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.758436:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.758439:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.758441:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.758456:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0253af lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x58d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.758457:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435079f3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+119:3627:x1870152993080064:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2730us (2861us total) trans 4294975684 rc 0/0 00000100:00100000:1.0:1783520935.758467:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad2d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993080448:192.168.204.33@tcp:104: 00000004:00080000:0.0:1783520935.758473:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435079f3480/2, rc 0, transno 4294975684 00000100:00100000:3.0:1783520935.758474:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44185 00010000:00010000:2.0:1783520935.758478:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0253af lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x58d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.758484:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0253bd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x58f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.758485:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993080064:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.758487:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0253af lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x58d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.758507:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1783520935.758518:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993081088:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.758521:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0253af lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x58d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.758541:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.758548:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.758572:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993081088 00000100:00100000:1.0:1783520935.758577:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993081088 00010000:00010000:0.0:1783520935.758588:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.758595:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0253af lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x58d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.758601:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.758611:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44189 00010000:00010000:0.0:1783520935.758612:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.758619:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0253af lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x58d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.758623:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515345500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+120:3627:x1870152993081088:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.758635:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435079f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+120:3627:x1870152993079552:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3440us (3751us total) trans 4294975685 rc 0/0 00010000:00010000:0.0:1783520935.758636:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0253af lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x58d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.758643:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1426 count 1 00000100:00100000:2.0:1783520935.758652:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44184 00000100:00100000:3.0:1783520935.758654:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.758660:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0253d2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x592:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.758668:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0253af lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x58d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.758676:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005645ad09/0xd5401cbabc01f7d6 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x592:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859eff52 expref: 879 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.758680:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0253af lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x58d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520935.758690:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435079f1f80/2, rc 0, transno 4294975685 00010000:00010000:1.0:1783520935.758695:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x592:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.758701:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x592:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.758705:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993079552:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.758707:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0253d2 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x592:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.758708:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.758713:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.758716:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.758736:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005645ad09/0xd5401cbabc01f7d6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x592:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff52 expref: 879 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.758739:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002376960 00000100:00100000:0.0:1783520935.758745:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002376960 00000100:00100000:2.0:1783520935.758748:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002377600 00010000:00010000:1.0:1783520935.758748:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005645ad09/0xd5401cbabc01f7d6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x592:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff52 expref: 879 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.758754:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002377600 00000100:00100000:1.0:1783520935.758760:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993081472:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.758767:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000060f5294a/0xd5401cbabc01f7c1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x58c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff28 expref: 881 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.758778:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e8505347/0xd5401cbabc01fc6e lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x591:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff4b expref: 881 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.758785:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.758795:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6f00 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.758818:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002377472 00000100:00100000:0.0:1783520935.758823:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002377472 00000100:00100000:1.0:1783520935.758826:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35778 00000100:00100000:2.0:1783520935.758834:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35779 00000100:00100000:1.0:1783520935.758840:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435079f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+882:2390:x1870153002376960:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.758842:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000011d75cc7/0xd5401cbabc01f7cf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x590:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff44 expref: 882 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.758850:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435079f0700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+882:2387:x1870153002377600:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.758850:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.758859:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000060f5294a/0xd5401cbabc01f7c1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x58c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff28 expref: 882 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.758866:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520935.758870:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.758876:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e8505347/0xd5401cbabc01fc6e lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x591:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff4b expref: 882 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.758876:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35780 00000100:00100000:0.0:1783520935.758891:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435079f1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+882:2390:x1870153002377472:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.758899:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000060f5294a/0xd5401cbabc01f7c1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x58c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff28 expref: 882 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.758903:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e8505347/0xd5401cbabc01fc6e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x591:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff4b expref: 882 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.758905:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.758914:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000011d75cc7/0xd5401cbabc01f7cf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x590:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff44 expref: 882 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.758931:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515344380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993080960:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.758932:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000060f5294a/0xd5401cbabc01f7c1 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x58c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff28 expref: 882 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.758934:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000011d75cc7/0xd5401cbabc01f7cf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x590:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff44 expref: 882 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.758937:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e8505347/0xd5401cbabc01fc6e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x591:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff4b expref: 882 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.758947:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0253cb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x591:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.758964:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x58c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.758968:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x58c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.758971:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x591:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.758971:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000011d75cc7/0xd5401cbabc01f7cf lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x590:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff44 expref: 880 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.758974:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0253a8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x58c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.758977:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002377344 00010000:00010000:2.0:1783520935.758981:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x591:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.758982:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002377344 00010000:00010000:2.0:1783520935.758992:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0253cb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x591:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.758997:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000831ccd3b/0xd5401cbabc01fc67 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x58f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff3d expref: 880 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.758999:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x590:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.759001:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0253a8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x58c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.759007:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x590:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.759012:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0253cb lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x591:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.759017:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0253c4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x590:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.759036:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.759038:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0253c4 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x590:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.759059:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.759066:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435079f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+880:2390:x1870153002376960:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 231us (467us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.759069:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.759078:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35778 00000100:00100000:2.0:1783520935.759097:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435079f0700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+879:2387:x1870153002377600:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 251us (445us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.759104:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35781 00000100:00100000:0.0:1783520935.759104:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435079f1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+879:2390:x1870153002377472:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 214us (322us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.759116:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35779 00000100:00100000:1.0:1783520935.759119:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435079f1500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+879:2390:x1870153002377344:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.759124:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0253cb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x591:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.759125:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35780 00010000:00010000:1.0:1783520935.759131:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.759138:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0253cb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x591:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.759141:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000831ccd3b/0xd5401cbabc01fc67 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x58f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff3d expref: 877 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.759151:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993081216:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.759160:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0253cb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x591:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.759160:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94361103b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993081344:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.759182:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000831ccd3b/0xd5401cbabc01fc67 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x58f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff3d expref: 877 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.759185:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0253cb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x591:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.759190:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.759205:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0253c4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x590:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.759208:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000831ccd3b/0xd5401cbabc01fc67 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x58f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff3d expref: 877 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.759217:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.759228:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0253c4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x590:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.759229:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x58f:0x0].0x0 since=0s v=(0 0) 00000004:00080000:2.0:1783520935.759230:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350d165c00/2, rc 0, transno 4294975682 00010000:00010000:1.0:1783520935.759239:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x58f:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.759245:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350d165c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993079168:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.759245:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0253c4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x590:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.759247:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0253bd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x58f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.759267:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0253c4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x590:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.759269:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0253bd lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x58f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.759292:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94361103bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993081600:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.759293:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:3.0:1783520935.759301:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943611038e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+120:3627:x1870152993080832:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2703us (2761us total) trans 4294975686 rc 0/0 00010000:00010000:1.0:1783520935.759302:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.759311:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44188 00000100:00100000:3.0:1783520935.759327:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993081216 00000100:00100000:3.0:1783520935.759333:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993081216 00000100:00100000:2.0:1783520935.759333:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.759333:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435079f1500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+876:2390:x1870153002377344:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 217us (466us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.759346:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35781 00000004:00080000:2.0:1783520935.759396:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435079f2300/2, rc 0, transno 4294975686 00000100:00100000:3.0:1783520935.759402:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993081600 00000100:00100000:0.0:1783520935.759403:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515344000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+121:3626:x1870152993080576:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3135us (3207us total) trans 4294975687 rc 0/0 00000100:00100000:3.0:1783520935.759408:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993081600 00000100:00100000:2.0:1783520935.759410:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993080832:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.759415:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00000100:00100000:0.0:1783520935.759416:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44187 00010000:00010000:1.0:1783520935.759422:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0253cb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x591:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.759433:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.759437:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0253cb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x591:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.759445:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44190 00000100:00100000:2.0:1783520935.759453:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1783520935.759455:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44191 00010000:00010000:1.0:1783520935.759457:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0253cb lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x591:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.759460:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94361103b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+121:3625:x1870152993081216:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.759465:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94361103a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993081728:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.759471:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943611038a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+121:3627:x1870152993081600:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520935.759480:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1427 count 1 00010000:00010000:1.0:1783520935.759487:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0253cb lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x591:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.759491:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1429 count 1 00010000:00010000:3.0:1783520935.759495:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc0253d9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x593:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.759515:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0253cb lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x591:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.759518:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0253e0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x595:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.759519:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.759533:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000000ace12a4/0xd5401cbabc01fc7c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x593:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859eff59 expref: 875 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.759547:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x593:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.759549:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b0b2f1f1/0xd5401cbabc01fc75 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x595:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859eff67 expref: 875 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.759552:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x593:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.759554:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993081728 00010000:00010000:1.0:1783520935.759554:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:2.0:1783520935.759560:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993081728 00010000:00010000:1.0:1783520935.759563:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0253c4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x590:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.759567:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x595:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.759574:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x595:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.759578:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.759580:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0253e0 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x595:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.759581:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44192 00010000:00010000:1.0:1783520935.759586:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0253c4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x590:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.759591:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943611039500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+122:3627:x1870152993081728:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.759604:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0253c4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x590:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.759611:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1430 count 1 00010000:00010000:0.0:1783520935.759613:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b0b2f1f1/0xd5401cbabc01fc75 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x595:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff67 expref: 875 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.759616:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0253c4 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x590:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.759626:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0253e7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x596:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.759632:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b0b2f1f1/0xd5401cbabc01fc75 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x595:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff67 expref: 875 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.759635:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc0253d9 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x593:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.759637:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0253c4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x590:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.759660:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006a53a995/0xd5401cbabc01f7e4 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x596:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859eff6e expref: 875 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.759664:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515347480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993081856:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.759680:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000000ace12a4/0xd5401cbabc01fc7c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x593:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff59 expref: 875 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.759681:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x596:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.759685:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x596:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.759691:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0253e7 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x596:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.759691:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.759696:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.759702:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000000ace12a4/0xd5401cbabc01fc7c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x593:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff59 expref: 875 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.759703:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.759710:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470d80 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.759715:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006a53a995/0xd5401cbabc01f7e4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x596:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff6e expref: 875 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.759723:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f3b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993081984:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.759728:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006a53a995/0xd5401cbabc01f7e4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x596:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff6e expref: 875 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.759729:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.759730:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0253bd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x58f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.759750:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872f00 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.759753:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.759754:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0253bd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x58f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.759757:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94361103aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993082112:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.759769:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0253bd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x58f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.759770:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993081344 00000100:00100000:3.0:1783520935.759775:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993081344 00010000:00010000:1.0:1783520935.759777:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0253bd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x58f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.759789:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94361103ad80/2, rc 0, transno 4294975687 00010000:00010000:1.0:1783520935.759793:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520935.759800:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f000 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.759803:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94361103ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993080576:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.759813:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44193 00010000:00010000:2.0:1783520935.759817:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.759823:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0253bd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x58f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.759824:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515344380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+123:3626:x1870152993081344:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.759838:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6e00 already unused, calling callback (ffffffffc0bf9e90) 00002000:00080000:3.0:1783520935.759843:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1428 count 1 00010000:00010000:2.0:1783520935.759844:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0253bd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x58f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.759855:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:2.0:1783520935.759856:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0253bd lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x58f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.759859:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0253ee lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x594:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.759865:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515346a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993082240:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.759877:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943611038700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+124:3626:x1870152993080192:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3907us (4095us total) trans 4294975688 rc 0/0 00010000:00010000:3.0:1783520935.759889:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008b809d9e/0xd5401cbabc01f7dd lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x594:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859eff60 expref: 875 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.759894:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44186 00010000:00010000:2.0:1783520935.759898:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0253bd lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x58f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.759904:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.759909:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x594:0x0].0x0 since=37s v=(0 0) 00000100:00100000:1.0:1783520935.759915:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993082240 00010000:00010000:3.0:1783520935.759918:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x594:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.759919:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000be742370/0xd5401cbabc0253bd lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x58f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.759921:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993082240 00010000:00010000:3.0:1783520935.759927:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0253ee lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x594:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.759938:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943611039f80/2, rc 0, transno 4294975688 00010000:00010000:2.0:1783520935.759940:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.759943:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.759947:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.759947:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44194 00000100:00100000:0.0:1783520935.759951:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943611039f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993080192:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.759953:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008b809d9e/0xd5401cbabc01f7dd lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x594:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff60 expref: 875 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.759960:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515345180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+124:3626:x1870152993082240:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.759974:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008b809d9e/0xd5401cbabc01f7dd lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x594:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff60 expref: 875 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.759978:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1431 count 1 00010000:00010000:1.0:1783520935.759995:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0253f5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x597:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.759997:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f0000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993082368:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.760013:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515344700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993082496:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.760015:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000bc090fea/0xd5401cbabc01fc91 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x597:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859eff75 expref: 875 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.760024:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x597:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.760028:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x597:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.760033:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607eb8a80 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.760033:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0253f5 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x597:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.760046:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.760059:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000bc090fea/0xd5401cbabc01fc91 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x597:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff75 expref: 875 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.760070:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000bc090fea/0xd5401cbabc01fc91 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x597:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff75 expref: 875 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.760076:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993082496 00000100:00100000:2.0:1783520935.760080:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993082496 00000100:00100000:1.0:1783520935.760083:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993082624:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.760104:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44195 00000100:00100000:2.0:1783520935.760111:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638995500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+125:3626:x1870152993082496:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.760120:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6300 going to sleep for 11 seconds 00002000:00080000:2.0:1783520935.760122:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1432 count 1 00010000:00010000:2.0:1783520935.760132:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0253fc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x598:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.760134:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0253a8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x58c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.760150:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4fb3c69/0xd5401cbabc01f7eb lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x598:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859eff7c expref: 875 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.760150:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0253a8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x58c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.760159:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993081472:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.760173:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x598:0x0].0x0 since=37s v=(0 0) 00000100:00100000:3.0:1783520935.760176:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.760177:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0253a8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x58c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.760178:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x598:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.760181:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0253d2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x592:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.760183:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0253fc lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x598:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.760193:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002377728 00000100:00100000:3.0:1783520935.760198:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002377728 00010000:00010000:0.0:1783520935.760201:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0253a8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x58c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.760213:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005645ad09/0xd5401cbabc01f7d6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x592:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff52 expref: 876 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.760224:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.760225:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d4fb3c69/0xd5401cbabc01f7eb lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x598:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff7c expref: 876 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.760233:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.760241:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d4fb3c69/0xd5401cbabc01f7eb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x598:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff7c expref: 876 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.760241:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0253a8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x58c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.760268:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943638996680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993082752:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.760268:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.760275:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0253a8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x58c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.760278:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35782 00010000:00010000:1.0:1783520935.760290:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0253a8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x58c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.760291:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435079f2d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+876:2388:x1870153002377728:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.760302:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520935.760303:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f180 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.760308:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005645ad09/0xd5401cbabc01f7d6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x592:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff52 expref: 876 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.760310:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0253a8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x58c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.760315:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cc7c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+126:3625:x1870152993079296:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5127us (5571us total) trans 4294975689 rc 0/0 00000004:00080000:2.0:1783520935.760327:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350d164a80/2, rc 0, transno 4294975689 00010000:00010000:3.0:1783520935.760331:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005645ad09/0xd5401cbabc01f7d6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x592:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff52 expref: 876 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.760334:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44183 00010000:00010000:1.0:1783520935.760337:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0253a8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x58c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.760340:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350d164a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993079296:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.760397:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005645ad09/0xd5401cbabc01f7d6 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x592:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff52 expref: 876 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.760398:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.760401:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.760404:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.760413:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x592:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.760418:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x592:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.760423:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0253d2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x592:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.760431:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515345c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993082880:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.760436:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0253d2 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x592:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.760457:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.760462:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.760469:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0253d2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x592:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.760479:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0253d2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x592:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.760480:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435079f2d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+875:2388:x1870153002377728:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 191us (308us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.760493:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35782 00010000:00010000:1.0:1783520935.760510:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0253d2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x592:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.760524:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0253d2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x592:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.760542:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.760553:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.760559:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0253d2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x592:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.760568:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.760571:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0253d2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x592:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.760576:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993082880 00010000:00010000:2.0:1783520935.760580:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0253d2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x592:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.760582:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993082880 00010000:00010000:2.0:1783520935.760597:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0253d2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x592:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.760604:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0253d2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x592:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.760621:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44196 00010000:00010000:2.0:1783520935.760624:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.760628:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515345500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+127:3627:x1870152993081088:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2006us (2077us total) trans 4294975690 rc 0/0 00010000:00010000:2.0:1783520935.760629:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.760633:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638997800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+127:3627:x1870152993082880:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.760634:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.760643:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44189 00002000:00080000:0.0:1783520935.760649:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1433 count 1 00010000:00010000:0.0:1783520935.760664:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc025403 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x599:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.760675:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ad3b80/2, rc 0, transno 4294975690 00010000:00010000:0.0:1783520935.760677:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000278c1780/0xd5401cbabc01fc83 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x599:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859eff83 expref: 874 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.760687:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x599:0x0].0x0 since=37s v=(0 0) 00000100:00100000:1.0:1783520935.760690:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993081088:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.760692:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x599:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.760700:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc025403 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x599:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.760723:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000278c1780/0xd5401cbabc01fc83 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x599:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff83 expref: 874 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.760734:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943638994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993083008:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.760735:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000278c1780/0xd5401cbabc01fc83 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x599:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff83 expref: 874 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.760750:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515346680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993083136:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.760769:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.760799:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993083008 00000100:00100000:0.0:1783520935.760802:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470cc0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.760804:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993083008 00000100:00100000:2.0:1783520935.760833:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44197 00000100:00100000:2.0:1783520935.760842:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+127:3628:x1870152993083008:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.760853:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1434 count 1 00010000:00010000:2.0:1783520935.760864:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02540a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x59a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.760875:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001a767443/0xd5401cbabc02285f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x59a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f4f expref: 874 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.760883:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x59a:0x0].0x0 since=36s v=(0 0) 00010000:00010000:2.0:1783520935.760885:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x59a:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.760889:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02540a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x59a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.760908:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001a767443/0xd5401cbabc02285f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x59a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f4f expref: 874 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.760918:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001a767443/0xd5401cbabc02285f lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x59a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f4f expref: 874 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.760928:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943638994a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993083264:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.760959:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff94363865b240 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.761605:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515347480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993081856:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.761620:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0253e0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x595:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.761920:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94361103aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993082112:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.761937:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0253e7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x596:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.763148:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.763183:0:17749:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002377984 00000100:00100000:0.0:1783520935.763188:0:17749:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002377984 00000100:00100000:3.0:1783520935.763232:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f0000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993082368:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.763233:0:17749:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98402 00000100:00100000:0.0:1783520935.763249:0:17749:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435079f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+21:251795:x1870153002377984:12345-192.168.204.33@tcp:101:dd.0 00010000:00010000:3.0:1783520935.763259:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0253ee lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x594:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.763264:0:17749:0:(ldlm_lockd.c:1265:ldlm_handle_enqueue()) ### server-side enqueue handler START 00010000:00010000:0.0:1783520935.763268:0:17749:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 2 locks, starting at 1 00000100:00100000:2.0:1783520935.763269:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993082624:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.763292:0:17749:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7e00/0xd5401cbabc0251fd lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x44a01000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34ab expref: 21 pid: 35280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.763293:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0253f5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x597:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.763307:0:17749:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x0 (ffff943504b72c00) 00010000:00010000:0.0:1783520935.763312:0:17749:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.763326:0:17749:0:(ldlm_lockd.c:1343:ldlm_handle_enqueue()) ### server-side enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfa400/0xd5401cbabc025411 lrc: 2/0,0 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT gid 0 flags: 0x40000000000000 nid: local remote: 0xc275876b859f34b9 expref: -99 pid: 17749 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.763333:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.763382:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002378624 00000100:00100000:3.0:1783520935.763388:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943638994a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993083264:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.763388:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002378624 00010000:00010000:2.0:1783520935.763411:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d4fb3c69/0xd5401cbabc01f7eb lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x598:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff7c expref: 875 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.763413:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02540a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x59a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.763436:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.763462:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002378240 00000100:00100000:1.0:1783520935.763467:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.763468:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002378240 00010000:00010000:2.0:1783520935.763484:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008b809d9e/0xd5401cbabc01f7dd lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x594:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff60 expref: 876 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.763510:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002378496 00000100:00100000:1.0:1783520935.763512:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.763515:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002378496 00000100:00100000:2.0:1783520935.763529:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002378752 00000100:00100000:2.0:1783520935.763534:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002378752 00010000:00010000:3.0:1783520935.763540:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000000ace12a4/0xd5401cbabc01fc7c lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x593:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff59 expref: 877 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.763554:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000278c1780/0xd5401cbabc01fc83 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x599:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff83 expref: 878 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.763554:0:17749:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbe00/0xd5401cbabc025418 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x40000000000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.763555:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.763568:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002378112 00010000:00010000:0.0:1783520935.763570:0:17749:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbe00/0xd5401cbabc025418 lrc: 3/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x50210000000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.763574:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002378112 00000100:00100000:2.0:1783520935.763579:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35783 00010000:00010000:0.0:1783520935.763583:0:17749:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbe00/0xd5401cbabc025418 lrc: 3/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40210000000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.763587:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.763594:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad1c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+879:2387:x1870153002378624:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.763596:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006a53a995/0xd5401cbabc01f7e4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x596:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff6e expref: 879 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.763605:0:17749:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbc00/0xd5401cbabc02541f lrc: 3/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x40000000000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.763606:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.763615:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d4fb3c69/0xd5401cbabc01f7eb lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x598:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff7c expref: 879 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.763619:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.763624:0:17749:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbc00/0xd5401cbabc02541f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.763633:0:17749:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbc00/0xd5401cbabc02541f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210000000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.763636:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d4fb3c69/0xd5401cbabc01f7eb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x598:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff7c expref: 879 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.763638:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002378368 00000100:00100000:3.0:1783520935.763642:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002378368 00010000:00010000:3.0:1783520935.763657:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000bc090fea/0xd5401cbabc01fc91 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x597:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff75 expref: 880 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.763658:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x598:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.763668:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x598:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.763671:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.763675:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0253fc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x598:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.763687:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002378880 00010000:00010000:2.0:1783520935.763689:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0253fc lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x598:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.763692:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002378880 00000100:00100000:3.0:1783520935.763695:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002379008 00010000:00010000:0.0:1783520935.763699:0:17749:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfa000/0xd5401cbabc025426 lrc: 3/1,0 mode: --/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x40000000000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.763701:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002379008 00010000:00010000:0.0:1783520935.763711:0:17749:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfa000/0xd5401cbabc025426 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT flags: 0x50210000000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.763715:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.763715:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001a767443/0xd5401cbabc02285f lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x59a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f4f expref: 882 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.763722:0:17749:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfa000/0xd5401cbabc025426 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT flags: 0x40210000000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.763723:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b0b2f1f1/0xd5401cbabc01fc75 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x595:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff67 expref: 882 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00200000:0.0:1783520935.763744:0:17749:0:(mdt_handler.c:909:mdt_pack_attr2body()) [0x200000402:0x72a:0x0]: returning size 131072 00000100:00100000:1.0:1783520935.763748:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35784 00000100:00100000:2.0:1783520935.763755:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad1c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+882:2387:x1870153002378624:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 164us (426us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.763761:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357d050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+882:2390:x1870153002378240:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520935.763767:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35785 00010000:00010000:1.0:1783520935.763770:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.763778:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008b809d9e/0xd5401cbabc01f7dd lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x594:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff60 expref: 882 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.763779:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+882:2387:x1870153002378496:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.763779:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35783 00010000:00010000:3.0:1783520935.763793:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.763796:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008b809d9e/0xd5401cbabc01f7dd lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x594:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff60 expref: 881 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.763800:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000000ace12a4/0xd5401cbabc01fc7c lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x593:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff59 expref: 881 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.763807:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35786 00010000:00010000:0.0:1783520935.763815:0:17749:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbe00/0xd5401cbabc025418 lrc: 4/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40210000000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.763821:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad1880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+881:2390:x1870153002378752:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.763825:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000000ace12a4/0xd5401cbabc01fc7c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x593:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff59 expref: 881 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.763831:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.763831:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008b809d9e/0xd5401cbabc01f7dd lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x594:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff60 expref: 881 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.763841:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000278c1780/0xd5401cbabc01fc83 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x599:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff83 expref: 881 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.763841:0:17749:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbe00/0xd5401cbabc025418 lrc: 4/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x50210400000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.763853:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x593:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.763858:0:17749:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbe00/0xd5401cbabc025418 lrc: 3/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x50210400000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.763860:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x593:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.763862:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x594:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.763866:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000278c1780/0xd5401cbabc01fc83 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x599:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff83 expref: 880 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.763869:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc0253d9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x593:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.763871:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x594:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.763873:0:17749:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbe00/0xd5401cbabc025418 lrc: 4/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40210400000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.763881:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0253ee lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x594:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.763893:0:17749:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfbe00 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520935.763894:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc0253d9 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x593:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.763898:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0253ee lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x594:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.763901:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x599:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.763901:0:17749:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbe00/0xd5401cbabc025418 lrc: 4/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40210400000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.763910:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x599:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.763920:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc025403 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x599:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.763921:0:17749:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbe00/0xd5401cbabc025418 lrc: 5/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x40218400000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.763928:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.763930:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc025403 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x599:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.763933:0:17749:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x0 (ffff943504b72c00) 00010000:00010000:0.0:1783520935.763937:0:17749:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:1.0:1783520935.763938:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.763941:0:17749:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbe00/0xd5401cbabc025418 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x44a19400000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.763945:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.763948:0:17749:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbe00/0xd5401cbabc025418 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x44a19400000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.763955:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+880:2387:x1870153002378496:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 178us (487us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.763962:0:17749:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbc00/0xd5401cbabc02541f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.763967:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35785 00000100:00100000:1.0:1783520935.763970:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357d050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+880:2390:x1870153002378240:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 210us (534us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.763973:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad1880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+880:2390:x1870153002378752:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 152us (474us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.763979:0:17749:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbc00/0xd5401cbabc02541f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210400000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.763986:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35787 00010000:00010000:0.0:1783520935.763993:0:17749:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbc00/0xd5401cbabc02541f lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.763995:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35784 00000100:00100000:3.0:1783520935.763997:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+879:2390:x1870153002378112:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.764007:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35786 00010000:00010000:0.0:1783520935.764009:0:17749:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfbc00 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520935.764011:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.764012:0:17749:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbc00/0xd5401cbabc02541f lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.764018:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006a53a995/0xd5401cbabc01f7e4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x596:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff6e expref: 877 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.764023:0:17749:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbc00/0xd5401cbabc02541f lrc: 4/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40218400000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.764030:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35788 00010000:00010000:3.0:1783520935.764032:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006a53a995/0xd5401cbabc01f7e4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x596:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff6e expref: 877 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.764035:0:17749:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x5ba6781 (ffff943515473f00) 00010000:00010000:0.0:1783520935.764040:0:17749:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00000100:00100000:2.0:1783520935.764041:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad0700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+877:2390:x1870153002378368:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.764044:0:17749:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbc00/0xd5401cbabc02541f lrc: 2/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.764047:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006a53a995/0xd5401cbabc01f7e4 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x596:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff6e expref: 877 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.764053:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35789 00010000:00010000:2.0:1783520935.764054:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.764061:0:17749:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfbc00/0xd5401cbabc02541f lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.764064:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000bc090fea/0xd5401cbabc01fc91 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x597:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff75 expref: 876 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.764067:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x596:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.764071:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+876:2390:x1870153002378880:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.764075:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x596:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.764082:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0253e7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x596:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.764090:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000bc090fea/0xd5401cbabc01fc91 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x597:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff75 expref: 876 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.764093:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0253e7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x596:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.764094:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.764100:0:17749:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfa400/0xd5401cbabc025411 lrc: 0/0,0 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x44000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34b9 expref: 23 pid: 17749 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.764104:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001a767443/0xd5401cbabc02285f lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x59a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f4f expref: 876 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.764117:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000bc090fea/0xd5401cbabc01fc91 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x597:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff75 expref: 876 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.764129:0:17749:0:(ldlm_lockd.c:1513:ldlm_handle_enqueue()) ### server-side enqueue handler, sending reply (err=0, rc=0) ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfa000/0xd5401cbabc025426 lrc: 3/0,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x40200000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34b9 expref: 22 pid: 17749 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.764137:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001a767443/0xd5401cbabc02285f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x59a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f4f expref: 876 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.764139:0:17749:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x72a:0x0].0x0 (ffff943515473e00) 00010000:00010000:2.0:1783520935.764140:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x597:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.764145:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.764145:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x597:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.764145:0:17749:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 1b 00010000:00010000:0.0:1783520935.764151:0:17749:0:(ldlm_lockd.c:1592:ldlm_handle_enqueue()) ### server-side enqueue handler END (lock ffff94351cbfa000, rc 0) 00010000:00010000:2.0:1783520935.764154:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0253f5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x597:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.764162:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0253f5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x597:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.764166:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001a767443/0xd5401cbabc02285f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x59a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f4f expref: 875 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.764173:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+875:2390:x1870153002378112:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 176us (619us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.764183:0:17749:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435079f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+22:251795:x1870153002377984:12345-192.168.204.33@tcp:101:dd.0 Request processed in 936us (1182us total) trans 8589957892 rc 0/0 00010000:00010000:2.0:1783520935.764185:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.764194:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35787 00010000:00010000:1.0:1783520935.764195:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x59a:0x0].0x0 since=0s v=(1 0) 00000100:00100000:0.0:1783520935.764199:0:17749:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98402 00010000:00010000:1.0:1783520935.764202:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x59a:0x0].0x0 since=0s v=(1 0) 00000100:00100000:2.0:1783520935.764204:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad0700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+874:2390:x1870153002378368:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 167us (622us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.764209:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02540a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x59a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.764216:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35788 00010000:00010000:1.0:1783520935.764224:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02540a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x59a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.764230:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35790 00000100:00100000:3.0:1783520935.764242:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cba3b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+872:2388:x1870153002379008:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.764249:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0253e7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x596:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.764250:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.764253:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.764257:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b0b2f1f1/0xd5401cbabc01fc75 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x595:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff67 expref: 872 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.764264:0:35275:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000278c1780/0xd5401cbabc01fc83 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x599:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff83 expref: 872 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.764268:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0253e7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x596:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.764280:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b0b2f1f1/0xd5401cbabc01fc75 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x595:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff67 expref: 872 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.764282:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0253e7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x596:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.764282:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+871:2390:x1870153002378880:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 214us (666us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.764286:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515346680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993083136:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.764301:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0253e7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x596:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.764304:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b0b2f1f1/0xd5401cbabc01fc75 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x595:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff67 expref: 871 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.764304:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35789 00010000:00010000:0.0:1783520935.764307:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc025403 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x599:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.764330:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x595:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.764330:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc025403 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x599:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.764332:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.764338:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x595:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.764345:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0253e0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x595:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.764347:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc025403 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x599:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.764353:0:8408:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000ace12a4/0xd5401cbabc01fc7c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x593:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff59 expref: 869 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.764381:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0253e0 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x595:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.764395:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc025403 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x599:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.764408:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f3b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993081984:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.764421:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc025403 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x599:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.764431:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc0253d9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x593:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.764439:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.764440:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc0253d9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x593:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.764440:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520935.764449:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc0253d9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x593:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.764455:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc0253d9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x593:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.764461:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc0253d9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x593:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.764466:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cba3b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+868:2388:x1870153002379008:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 223us (798us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.764483:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35790 00000100:00100000:2.0:1783520935.764485:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943611039500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+129:3627:x1870152993081728:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4892us (4967us total) trans 4294975691 rc 0/0 00010000:00010000:1.0:1783520935.764488:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to regular blp list, 3 blwis in pool 00000100:00100000:2.0:1783520935.764514:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44192 00010000:00010000:3.0:1783520935.764529:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0253ee lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x594:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.764532:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638997800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+128:3627:x1870152993082880:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3903us (4070us total) trans 4294975692 rc 0/0 00010000:00010000:2.0:1783520935.764539:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:2.0:1783520935.764546:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0253e7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x596:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.764548:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44196 00010000:00010000:3.0:1783520935.764550:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0253ee lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x594:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.764562:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.764565:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520935.764567:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0253ee lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x594:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.764569:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0253e7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x596:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.764570:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc025403 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x599:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.764584:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94361103b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+128:3625:x1870152993081216:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5128us (5398us total) trans 4294975693 rc 0/0 00010000:00010000:3.0:1783520935.764592:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0253ee lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x594:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.764594:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0253e7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x596:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.764599:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44190 00010000:00010000:0.0:1783520935.764610:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfa600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.764616:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc025403 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x599:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.764621:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0253e7 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x596:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.764625:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc025403 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x599:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.764627:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520935.764628:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0253e7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x596:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.764629:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1783520935.764638:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc0253d9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x593:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.764641:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc025403 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x599:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.764648:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.764653:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0253ee lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x594:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.764661:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.764662:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc025403 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x599:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.764663:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.764671:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0253ee lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x594:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.764671:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc0253d9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x593:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.764677:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.764681:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.764685:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.764686:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0253ee lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x594:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.764709:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc0253d9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x593:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.764711:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0253ee lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x594:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.764724:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0253ee lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x594:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.764725:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc0253d9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x593:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.764727:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515344380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+128:3626:x1870152993081344:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4904us (4975us total) trans 4294975694 rc 0/0 00000004:00080000:0.0:1783520935.764742:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94361103b480/2, rc 0, transno 4294975694 00000100:00100000:3.0:1783520935.764748:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44193 00010000:00010000:1.0:1783520935.764748:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc0253d9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x593:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.764758:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.764758:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94361103b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993081344:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.764761:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.764764:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.764773:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.764773:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.764776:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.764780:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:2.0:1783520935.764796:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515345c00/2, rc 0, transno 4294975692 00000100:00100000:2.0:1783520935.764803:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515345c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993082880:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.764811:0:8410:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4fb3c69/0xd5401cbabc01f7eb lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x598:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff7c expref: 867 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520935.764820:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94361103a300/2, rc 0, transno 4294975691 00000100:00100000:1.0:1783520935.764826:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943638996680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993082752:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.764831:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94361103a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993081728:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.764833:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515345880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993083520:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.764845:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0253fc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x598:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.764853:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0253fc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x598:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.764855:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0253e0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x595:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.764861:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0253fc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x598:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.764873:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0253e0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x595:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.764881:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.764884:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0253fc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x598:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.764889:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0253e0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x595:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.764891:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0253fc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x598:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.764910:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0253e0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x595:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.764915:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.764915:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1783520935.764923:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515346680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993083392:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.764926:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:0.0:1783520935.764950:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.764965:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02540a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x59a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.764975:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02540a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x59a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.764982:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520935.764990:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0253fc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x598:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.765000:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.765003:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0253fc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x598:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.765013:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638995500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+129:3626:x1870152993082496:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4899us (4963us total) trans 4294975695 rc 0/0 00010000:00010000:0.0:1783520935.765014:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0253fc lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x598:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.765018:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943611038a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+129:3627:x1870152993081600:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5551us (5687us total) trans 4294975696 rc 0/0 00000100:00100000:1.0:1783520935.765028:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44195 00010000:00010000:0.0:1783520935.765033:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0253fc lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x598:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.765034:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44191 00010000:00010000:3.0:1783520935.765040:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02540a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x59a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.765040:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0253fc lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x598:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.765045:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993083520 00000100:00100000:1.0:1783520935.765051:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993083520 00000100:00100000:2.0:1783520935.765052:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993083392 00000100:00100000:2.0:1783520935.765058:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993083392 00010000:00010000:3.0:1783520935.765061:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02540a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x59a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.765073:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520935.765076:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.765079:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44198 00010000:00010000:0.0:1783520935.765079:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0253e0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x595:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.765089:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfa200 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520935.765092:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44199 00000100:00100000:1.0:1783520935.765092:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515346d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+129:3626:x1870152993083520:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.765095:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0253e0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x595:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.765103:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515347480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+129:3626:x1870152993083392:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.765106:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0253e0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x595:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.765108:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1436 count 1 00002000:00080000:2.0:1783520935.765119:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1435 count 1 00010000:00010000:0.0:1783520935.765120:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0253e0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x595:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.765123:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025434 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x59c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.765132:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02543b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x59b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.765138:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0253e0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x595:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.765145:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+130:3628:x1870152993083008:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4303us (4375us total) trans 4294975697 rc 0/0 00010000:00010000:1.0:1783520935.765147:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d53878fd/0xd5401cbabc01f7f9 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x59c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859eff98 expref: 866 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.765155:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004636b622/0xd5401cbabc01fc8a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x59b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859eff91 expref: 866 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.765162:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44197 00010000:00010000:0.0:1783520935.765164:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.765167:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x59c:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.765170:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02540a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x59a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.765174:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x59c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.765182:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025434 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x59c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.765184:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.765186:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x59b:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.765190:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02540a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x59a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.765192:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x59b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.765200:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02543b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x59b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.765207:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02540a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x59a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.765221:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02540a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x59a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.765227:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02540a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x59a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.765231:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d53878fd/0xd5401cbabc01f7f9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x59c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff98 expref: 866 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.765236:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004636b622/0xd5401cbabc01fc8a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x59b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff91 expref: 866 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.765245:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.765248:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.765251:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.765254:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d53878fd/0xd5401cbabc01f7f9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x59c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff98 expref: 866 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.765255:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004636b622/0xd5401cbabc01fc8a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x59b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff91 expref: 866 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.765271:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993083648:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.765277:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943638994e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993083776:192.168.204.33@tcp:104: 00000004:00080000:0.0:1783520935.765290:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515344700/2, rc 0, transno 4294975695 00000100:00100000:0.0:1783520935.765298:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515344700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993082496:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.765576:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff94363865b240 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.765576:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6840 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.765593:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.765598:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.765603:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993084032:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.765612:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993083904:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.765640:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:2.0:1783520935.765671:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ad2d80/2, rc 0, transno 4294975693 00000004:00080000:1.0:1783520935.765683:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943638994000/2, rc 0, transno 4294975697 00000100:00100000:2.0:1783520935.765684:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993081216:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.765692:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943638994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993083008:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.765701:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993084032 00000100:00100000:3.0:1783520935.765707:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993084032 00010000:00010000:2.0:1783520935.765708:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0253f5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x597:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.765720:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0253f5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x597:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.765720:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.765737:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0253f5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x597:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.765739:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993083904 00000100:00100000:3.0:1783520935.765745:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993083904 00010000:00010000:2.0:1783520935.765748:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0253f5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x597:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.765754:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44200 00000004:00080000:1.0:1783520935.765764:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94361103bb80/2, rc 0, transno 4294975696 00010000:00010000:2.0:1783520935.765765:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.765765:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e0f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+131:3628:x1870152993084032:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.765775:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94361103bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993081600:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.765778:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44201 00002000:00080000:0.0:1783520935.765779:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1438 count 1 00000100:00100000:3.0:1783520935.765792:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cc7ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+131:3627:x1870152993083904:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.765795:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025442 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x59e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.765807:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a937986c/0xd5401cbabc01f800 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x59e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859effa6 expref: 866 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520935.765809:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1437 count 1 00010000:00010000:0.0:1783520935.765820:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x59e:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.765824:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x59e:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.765826:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.765828:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025449 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x59d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.765829:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025442 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x59e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.765833:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0253f5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x597:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.765840:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515345180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+132:3626:x1870152993082240:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5882us (5940us total) trans 4294975698 rc 0/0 00000100:00100000:2.0:1783520935.765858:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44194 00010000:00010000:1.0:1783520935.765858:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.765865:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0253f5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x597:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.765868:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a937986c/0xd5401cbabc01f800 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x59e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859effa6 expref: 866 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.765884:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517245500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993084160:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.765890:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a937986c/0xd5401cbabc01f800 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x59e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859effa6 expref: 866 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.765904:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0253f5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x597:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.765908:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517245c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993084544:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.765911:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.765927:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0253f5 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x597:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.765932:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000338b3661/0xd5401cbabc01fc98 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x59d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859eff9f expref: 866 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.765937:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1783520935.765939:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470cc0 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.765942:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0253f5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x597:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.765947:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517244700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993084288:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.765955:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x59d:0x0].0x0 since=37s v=(0 0) 00000004:00080000:0.0:1783520935.765961:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515346a00/2, rc 0, transno 4294975698 00010000:00010000:3.0:1783520935.765963:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x59d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.765969:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025449 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x59d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.765973:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515346a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993082240:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.765977:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.765978:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.765982:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.765988:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.766003:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000338b3661/0xd5401cbabc01fc98 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x59d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff9f expref: 866 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.766010:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993084160 00000100:00100000:2.0:1783520935.766015:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993084160 00010000:00010000:3.0:1783520935.766016:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000338b3661/0xd5401cbabc01fc98 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x59d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff9f expref: 866 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.766023:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514f9b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993084800:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.766034:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f0000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993084672:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.766043:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993084288 00000100:00100000:2.0:1783520935.766048:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993084288 00000100:00100000:1.0:1783520935.766052:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.766056:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44202 00000100:00100000:0.0:1783520935.766067:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638994700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+133:3625:x1870152993084160:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.766069:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435129670c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.766073:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993084800 00000100:00100000:2.0:1783520935.766076:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993084800 00000100:00100000:1.0:1783520935.766077:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1783520935.766084:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cc7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993084416:0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.766084:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1439 count 1 00000100:00100000:3.0:1783520935.766096:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44203 00010000:00010000:0.0:1783520935.766100:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025450 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x59f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.766103:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44204 00000100:00100000:3.0:1783520935.766111:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638996a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+134:3625:x1870152993084288:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.766111:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.766117:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514f99c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+134:3628:x1870152993084800:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.766120:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004269b83a/0xd5401cbabc01fc9f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x59f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859effad expref: 866 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.766127:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x59f:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.766130:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x59f:0x0].0x0 since=0s v=(0 0) 00002000:00080000:3.0:1783520935.766132:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1440 count 1 00002000:00080000:2.0:1783520935.766132:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1442 count 1 00010000:00010000:0.0:1783520935.766134:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025450 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x59f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.766135:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993084416 00000100:00100000:1.0:1783520935.766142:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993084416 00010000:00010000:2.0:1783520935.766149:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025457 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5a2:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.766151:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02545e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5a0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.766161:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004269b83a/0xd5401cbabc01fc9f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x59f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859effad expref: 866 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.766169:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44205 00010000:00010000:2.0:1783520935.766176:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000012c9f7d8/0xd5401cbabc01f87e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f00a2 expref: 866 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.766184:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a2:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.766187:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a2:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.766187:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514f9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+135:3628:x1870152993084416:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.766188:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009296ead9/0xd5401cbabc01f807 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859effb4 expref: 866 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.766188:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004269b83a/0xd5401cbabc01fc9f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x59f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859effad expref: 866 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.766194:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025457 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.766209:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1441 count 1 00010000:00010000:3.0:1783520935.766216:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a0:0x0].0x0 since=37s v=(0 0) 00000100:00100000:0.0:1783520935.766217:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517244000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993084928:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.766223:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.766227:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025465 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5a1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.766230:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02545e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.766230:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000012c9f7d8/0xd5401cbabc01f87e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00a2 expref: 866 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.766244:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470f00 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.766258:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000012c9f7d8/0xd5401cbabc01f87e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00a2 expref: 866 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.766266:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002399bdc2/0xd5401cbabc01fca6 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859effbb expref: 866 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.766270:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009296ead9/0xd5401cbabc01f807 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859effb4 expref: 866 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.766274:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943638995180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993085056:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.766287:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a1:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.766292:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.766294:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009296ead9/0xd5401cbabc01f807 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859effb4 expref: 866 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.766298:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025465 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.766303:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f000 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.766328:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002399bdc2/0xd5401cbabc01fca6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859effbb expref: 866 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.766338:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002399bdc2/0xd5401cbabc01fca6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859effbb expref: 866 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.766349:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c550e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993085312:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.766407:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f3b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993085184:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.766426:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6cc0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.766442:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435129679c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.766473:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993083648:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.766490:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025434 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x59c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.766582:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943638994e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993083776:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.766602:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02543b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x59b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.766624:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.766645:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002379136 00000100:00100000:2.0:1783520935.766649:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002379136 00010000:00010000:2.0:1783520935.766662:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d53878fd/0xd5401cbabc01f7f9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x59c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff98 expref: 867 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.766682:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35791 00000100:00100000:2.0:1783520935.766689:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357e050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+867:2387:x1870153002379136:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.766699:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.766704:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d53878fd/0xd5401cbabc01f7f9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x59c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff98 expref: 867 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.766714:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d53878fd/0xd5401cbabc01f7f9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x59c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff98 expref: 867 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.766728:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d53878fd/0xd5401cbabc01f7f9 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x59c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff98 expref: 867 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.766738:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x59c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.766741:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x59c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.766746:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025434 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x59c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.766755:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025434 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x59c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.766773:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.766781:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025434 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x59c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.766790:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025434 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x59c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.766798:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025434 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x59c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.766805:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025434 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x59c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.766813:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357e050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+866:2387:x1870153002379136:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 125us (190us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.766820:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35791 00010000:00010000:0.0:1783520935.766820:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.766830:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.766835:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025434 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x59c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.766841:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.766845:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025434 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x59c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.766852:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025434 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x59c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.766864:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025434 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x59c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.766873:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025434 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x59c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.766874:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.766894:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.766897:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.766898:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.766900:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002379264 00000100:00100000:2.0:1783520935.766904:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002379264 00010000:00010000:2.0:1783520935.766918:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004636b622/0xd5401cbabc01fc8a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x59b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff91 expref: 866 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.766947:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35792 00000100:00100000:2.0:1783520935.766955:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435079f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+866:2387:x1870153002379264:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.766962:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.766966:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004636b622/0xd5401cbabc01fc8a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x59b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff91 expref: 866 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.766976:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004636b622/0xd5401cbabc01fc8a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x59b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff91 expref: 866 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.766989:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004636b622/0xd5401cbabc01fc8a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x59b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff91 expref: 866 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.766993:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515346d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+136:3626:x1870152993083520:12345-0@lo:6:osp-syn-1-0.0 Request processed in 1906us (2114us total) trans 4294975699 rc 0/0 00010000:00010000:2.0:1783520935.766999:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x59b:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.766999:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44198 00010000:00010000:2.0:1783520935.767002:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x59b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.767006:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02543b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x59b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.767014:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02543b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x59b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.767033:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515345880/2, rc 0, transno 4294975699 00010000:00010000:2.0:1783520935.767035:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.767050:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515345880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993083520:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.767063:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02543b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x59b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.767067:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435079f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+865:2387:x1870153002379264:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 110us (192us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.767083:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02543b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x59b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.767085:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35792 00010000:00010000:3.0:1783520935.767098:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02543b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x59b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.767098:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.767109:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02543b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x59b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.767111:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517247b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993085440:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.767125:0:6494:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002379392 00000100:00100000:0.0:1783520935.767130:0:6494:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002379392 00010000:00010000:3.0:1783520935.767139:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520935.767148:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.767167:0:6494:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98403 00010000:00010000:2.0:1783520935.767176:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520935.767176:0:6494:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c551c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+22:251795:x1870153002379392:12345-192.168.204.33@tcp:101:dd.0 00010000:00010000:2.0:1783520935.767181:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02543b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x59b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.767188:0:6494:0:(ldlm_lockd.c:1265:ldlm_handle_enqueue()) ### server-side enqueue handler START 00010000:00010000:2.0:1783520935.767189:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.767192:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02543b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x59b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.767200:0:6494:0:(ldlm_lockd.c:1343:ldlm_handle_enqueue()) ### server-side enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfb600/0xd5401cbabc02546c lrc: 2/0,0 mode: --/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT gid 0 flags: 0x40000000000000 nid: local remote: 0xc275876b859f34c0 expref: -99 pid: 6494 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.767201:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02543b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x59b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.767214:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02543b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x59b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.767220:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02543b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x59b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.767224:0:6494:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfba00/0xd5401cbabc025473 lrc: 3/1,0 mode: --/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x0/0x0 rrc: 6 type: IBT flags: 0x40000000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.767233:0:6494:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfba00/0xd5401cbabc025473 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x20/0x0 rrc: 6 type: IBT flags: 0x50210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.767241:0:6494:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfba00/0xd5401cbabc025473 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x20/0x0 rrc: 6 type: IBT flags: 0x40210000000000 pid: 6494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.767242:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.767245:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.767247:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.767274:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515347480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+136:3626:x1870152993083392:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2173us (2323us total) trans 4294975700 rc 0/0 00000100:00100000:3.0:1783520935.767283:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44199 00000004:00080000:1.0:1783520935.767288:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515346680/2, rc 0, transno 4294975700 00000100:00100000:3.0:1783520935.767299:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993085440 00000100:00100000:1.0:1783520935.767303:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515346680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993083392:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.767304:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993085440 00010000:00010000:0.0:1783520935.767329:0:6494:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfb600/0xd5401cbabc02546c lrc: 0/0,0 mode: --/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x20/0x0 rrc: 6 type: IBT gid 0 flags: 0x44000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34c0 expref: 24 pid: 6494 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.767339:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44206 00000100:00100000:1.0:1783520935.767346:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943638994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993085568:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.767347:0:6494:0:(ldlm_lockd.c:1513:ldlm_handle_enqueue()) ### server-side enqueue handler, sending reply (err=0, rc=0) ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfba00/0xd5401cbabc025473 lrc: 3/0,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x20/0x0 rrc: 5 type: IBT gid 0 flags: 0x40200000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34c0 expref: 23 pid: 6494 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.767352:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638996d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+136:3625:x1870152993085440:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.767391:0:6494:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x72a:0x0].0x0 (ffff943515473e00) 00010000:00010000:0.0:1783520935.767395:0:6494:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 20 00010000:00010000:0.0:1783520935.767398:0:6494:0:(ldlm_lockd.c:1592:ldlm_handle_enqueue()) ### server-side enqueue handler END (lock ffff94351cbfba00, rc 0) 00002000:00080000:3.0:1783520935.767399:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1443 count 1 00010000:00010000:3.0:1783520935.767414:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02547a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5a3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.767414:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.767430:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b6e760f4/0xd5401cbabc01fd1d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f00a9 expref: 864 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.767432:0:6494:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c551c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+23:251795:x1870153002379392:12345-192.168.204.33@tcp:101:dd.0 Request processed in 256us (335us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.767436:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993085568 00000100:00100000:2.0:1783520935.767441:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993085568 00000100:00100000:0.0:1783520935.767448:0:6494:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98403 00010000:00010000:3.0:1783520935.767449:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a3:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.767454:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.767458:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02547a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.767466:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44207 00000100:00100000:2.0:1783520935.767474:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c552300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+137:3626:x1870152993085568:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.767484:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b6e760f4/0xd5401cbabc01fd1d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00a9 expref: 864 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.767488:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1444 count 1 00010000:00010000:3.0:1783520935.767518:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b6e760f4/0xd5401cbabc01fd1d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00a9 expref: 864 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.767518:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025481 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5a4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.767549:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993085696:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.767556:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e9610a18/0xd5401cbabc01f885 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f00b0 expref: 864 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.767565:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a4:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.767568:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.767572:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025481 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.767589:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e9610a18/0xd5401cbabc01f885 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00b0 expref: 864 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.767599:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e9610a18/0xd5401cbabc01f885 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00b0 expref: 864 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.767610:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943638996680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993085824:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.767640:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895840 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.767641:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967e40 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.768151:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f0000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993084672:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.768170:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025449 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x59d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.768179:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517244000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993084928:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.768187:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.768205:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025450 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x59f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.768207:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517245c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993084544:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.768212:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002379520 00000100:00100000:2.0:1783520935.768218:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.768220:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002379520 00010000:00010000:1.0:1783520935.768229:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025442 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x59e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.768244:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000338b3661/0xd5401cbabc01fc98 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x59d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff9f expref: 865 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.768257:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.768257:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002379776 00000100:00100000:0.0:1783520935.768262:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002379776 00010000:00010000:0.0:1783520935.768278:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a937986c/0xd5401cbabc01f800 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x59e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859effa6 expref: 866 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.768296:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002379648 00000100:00100000:0.0:1783520935.768298:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002379648 00010000:00010000:0.0:1783520935.768309:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004269b83a/0xd5401cbabc01fc9f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x59f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859effad expref: 867 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.768312:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35793 00000100:00100000:1.0:1783520935.768325:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cba3b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+867:2388:x1870153002379776:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.768332:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.768337:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a937986c/0xd5401cbabc01f800 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x59e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859effa6 expref: 867 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.768344:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943638995180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993085056:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.768344:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35794 00010000:00010000:1.0:1783520935.768353:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a937986c/0xd5401cbabc01f800 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x59e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859effa6 expref: 867 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.768389:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a937986c/0xd5401cbabc01f800 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x59e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859effa6 expref: 867 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.768398:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x59e:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.768401:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x59e:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.768405:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025442 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x59e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.768412:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35795 00010000:00010000:1.0:1783520935.768417:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025442 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x59e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.768429:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025457 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.768446:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.768448:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94361103aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+866:2389:x1870153002379648:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.768465:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.768471:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004269b83a/0xd5401cbabc01fc9f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x59f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859effad expref: 866 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.768477:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638994a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+866:2387:x1870153002379520:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.768480:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cba3b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+866:2388:x1870153002379776:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 156us (264us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.768494:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.768507:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35793 00010000:00010000:0.0:1783520935.768512:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004269b83a/0xd5401cbabc01fc9f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x59f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859effad expref: 866 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.768518:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000338b3661/0xd5401cbabc01fc98 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x59d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff9f expref: 866 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.768529:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025442 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x59e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.768539:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004269b83a/0xd5401cbabc01fc9f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x59f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859effad expref: 865 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.768540:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025442 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x59e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.768559:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025442 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x59e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.768564:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025442 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x59e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.768565:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x59f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.768574:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x59f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.768580:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025450 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x59f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.768583:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520935.768594:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.768595:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025450 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x59f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.768618:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.768620:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.768625:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025442 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x59e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.768635:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.768640:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025442 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x59e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.768652:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94361103aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+864:2389:x1870153002379648:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 295us (398us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.768654:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025442 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x59e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.768667:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35794 00010000:00010000:2.0:1783520935.768674:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025442 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x59e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.768681:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025442 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x59e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.768685:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002379904 00000100:00100000:0.0:1783520935.768691:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002379904 00000100:00100000:1.0:1783520935.768694:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e0f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+138:3628:x1870152993084032:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2930us (3052us total) trans 4294975701 rc 0/0 00010000:00010000:2.0:1783520935.768696:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.768698:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000338b3661/0xd5401cbabc01fc98 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x59d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff9f expref: 864 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.768700:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.768705:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.768708:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44200 00010000:00010000:0.0:1783520935.768709:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000012c9f7d8/0xd5401cbabc01f87e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00a2 expref: 864 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.768728:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000338b3661/0xd5401cbabc01fc98 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x59d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859eff9f expref: 864 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.768743:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x59d:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.768749:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.768750:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35796 00010000:00010000:3.0:1783520935.768752:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x59d:0x0].0x0 since=0s v=(0 0) 00000004:00080000:1.0:1783520935.768753:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e9c000/2, rc 0, transno 4294975701 00010000:00010000:3.0:1783520935.768760:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025449 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x59d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.768766:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638995180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+863:2387:x1870153002379904:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.768770:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025449 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x59d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.768771:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993084032:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.768783:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.768790:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000012c9f7d8/0xd5401cbabc01f87e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00a2 expref: 863 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.768803:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000012c9f7d8/0xd5401cbabc01f87e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5a2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00a2 expref: 863 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.768822:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000012c9f7d8/0xd5401cbabc01f87e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5a2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00a2 expref: 863 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.768824:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c550380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993085952:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.768839:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.768841:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.768846:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.768849:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.768851:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025457 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.768854:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.768864:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025457 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.768877:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.768888:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.768891:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002380160 00000100:00100000:1.0:1783520935.768895:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002380160 00000100:00100000:2.0:1783520935.768907:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.768912:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009296ead9/0xd5401cbabc01f807 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859effb4 expref: 863 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.768920:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638994a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+863:2387:x1870153002379520:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 445us (733us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.768927:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638995180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+863:2387:x1870153002379904:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 164us (344us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.768936:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35795 00000100:00100000:1.0:1783520935.768936:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002380032 00000100:00100000:1.0:1783520935.768941:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002380032 00000100:00100000:0.0:1783520935.768947:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35796 00000100:00100000:3.0:1783520935.768958:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002380288 00010000:00010000:1.0:1783520935.768958:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002399bdc2/0xd5401cbabc01fca6 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859effbb expref: 863 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.768964:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002380288 00000100:00100000:0.0:1783520935.768970:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002380416 00000100:00100000:0.0:1783520935.768976:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002380416 00010000:00010000:3.0:1783520935.768984:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b6e760f4/0xd5401cbabc01fd1d lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00a9 expref: 864 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.768986:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35797 00010000:00010000:0.0:1783520935.768993:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e9610a18/0xd5401cbabc01f885 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00b0 expref: 864 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.768997:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35798 00000100:00100000:2.0:1783520935.769001:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505876300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+864:2387:x1870153002380160:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.769014:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357fc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+864:2387:x1870153002380032:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.769015:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.769025:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009296ead9/0xd5401cbabc01f807 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5a0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859effb4 expref: 864 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.769027:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.769043:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009296ead9/0xd5401cbabc01f807 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859effb4 expref: 864 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.769043:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002399bdc2/0xd5401cbabc01fca6 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5a1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859effbb expref: 864 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.769061:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a0:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.769062:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35799 00010000:00010000:2.0:1783520935.769066:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.769066:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002399bdc2/0xd5401cbabc01fca6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859effbb expref: 864 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.769072:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02545e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.769076:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35800 00000100:00100000:3.0:1783520935.769081:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505876680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+864:2387:x1870153002380288:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.769093:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02545e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5a0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.769094:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505877100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+864:2387:x1870153002380416:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.769095:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.769098:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.769107:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.769109:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b6e760f4/0xd5401cbabc01fd1d lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5a3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00a9 expref: 864 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.769112:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.769114:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.769116:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025465 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.769124:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e9610a18/0xd5401cbabc01f885 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5a4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00b0 expref: 864 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.769136:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b6e760f4/0xd5401cbabc01fd1d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00a9 expref: 864 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.769138:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505876300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+864:2387:x1870153002380160:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 141us (395us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.769143:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025465 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5a1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.769156:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35797 00010000:00010000:0.0:1783520935.769158:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e9610a18/0xd5401cbabc01f885 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00b0 expref: 864 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.769177:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025457 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.769180:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.769187:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.769189:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.769190:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025457 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.769199:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025481 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.769200:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025457 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.769213:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357fc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+863:2387:x1870153002380032:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 201us (373us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.769223:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025481 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5a4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.769230:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a3:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.769233:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35798 00010000:00010000:3.0:1783520935.769242:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.769244:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025457 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.769251:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02547a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.769255:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.769262:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.769265:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02547a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5a3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.769281:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505877100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+862:2387:x1870153002380416:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 192us (374us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.769284:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.769284:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.769292:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025457 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.769293:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35800 00010000:00010000:1.0:1783520935.769304:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.769308:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025457 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.769314:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505876680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+861:2387:x1870153002380288:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 237us (436us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.769320:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025450 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x59f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.769322:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025457 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.769328:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35799 00010000:00010000:0.0:1783520935.769337:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025450 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x59f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.769345:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025450 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x59f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.769346:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025457 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.769351:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025450 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x59f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.769355:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514f99c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+138:3628:x1870152993084800:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3241us (3302us total) trans 4294975702 rc 0/0 00010000:00010000:0.0:1783520935.769381:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.769388:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025457 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.769393:0:35282:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009296ead9/0xd5401cbabc01f807 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859effb4 expref: 860 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.769408:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:2.0:1783520935.769415:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44204 00010000:00010000:0.0:1783520935.769416:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025450 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x59f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.769419:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f3b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993085184:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.769420:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.769425:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.769431:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.769433:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb800 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520935.769436:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993085952 00010000:00010000:0.0:1783520935.769440:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025450 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x59f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.769441:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02545e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.769441:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993085952 00000004:00080000:1.0:1783520935.769456:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514f9b800/2, rc 0, transno 4294975702 00010000:00010000:3.0:1783520935.769466:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02545e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.769467:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44208 00010000:00010000:0.0:1783520935.769468:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025450 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x59f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.769473:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514f9b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993084800:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.769480:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c553800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+138:3627:x1870152993085952:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.769484:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02545e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.769510:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025450 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x59f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.769512:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02545e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.769515:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1445 count 1 00010000:00010000:3.0:1783520935.769533:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02545e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.769533:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025488 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5a5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.769535:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:0.0:1783520935.769539:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025450 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x59f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.769546:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c553b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993086080:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.769568:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007c33b957/0xd5401cbabc01fd24 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f00b7 expref: 859 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.769569:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.769573:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520935.769578:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.769580:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02545e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.769587:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a5:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.769594:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.769594:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.769600:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02545e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.769602:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025488 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.769625:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02545e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.769629:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.769637:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.769640:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02545e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.769649:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02545e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.769656:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007c33b957/0xd5401cbabc01fd24 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00b7 expref: 859 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.769669:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.769674:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007c33b957/0xd5401cbabc01fd24 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00b7 expref: 859 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.769674:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.769675:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638996a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+139:3625:x1870152993084288:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3566us (3695us total) trans 4294975703 rc 0/0 00010000:00010000:0.0:1783520935.769689:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.769695:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44203 00000100:00100000:2.0:1783520935.769700:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943505875f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993086208:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.769716:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993086080 00000100:00100000:2.0:1783520935.769719:0:35281:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002380672 00000100:00100000:3.0:1783520935.769722:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993086080 00000100:00100000:2.0:1783520935.769724:0:35281:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002380672 00000100:00100000:3.0:1783520935.769756:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44209 00010000:00010000:1.0:1783520935.769757:0:8408:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002399bdc2/0xd5401cbabc01fca6 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859effbb expref: 859 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.769767:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c551880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+140:3627:x1870152993086080:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.769776:0:35281:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98404 00000100:00100000:1.0:1783520935.769776:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c550e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993085312:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.769778:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638994700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+140:3625:x1870152993084160:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3711us (3865us total) trans 4294975704 rc 0/0 00000100:00100000:2.0:1783520935.769791:0:35281:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c551500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_012:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+23:251795:x1870153002380672:12345-192.168.204.33@tcp:36:dd.0 00000100:00100000:0.0:1783520935.769798:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44202 00010000:00010000:1.0:1783520935.769799:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025465 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.769810:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025465 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.769818:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025465 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.769826:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025465 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520935.769827:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1446 count 1 00010000:00010000:2.0:1783520935.769828:0:35281:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.769839:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025465 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.769844:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc02548f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5a6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.769846:0:35281:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbfa000/0xd5401cbabc025426 lrc: 0/0,0 mode: --/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x44a01000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34b9 expref: 23 pid: 17749 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.769858:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.769868:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.769873:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006b7bd685/0xd5401cbabc01f88c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f00be expref: 858 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.769874:0:35281:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x72a:0x0].0x0 (ffff943515473e00) 00010000:00010000:0.0:1783520935.769875:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025465 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.769882:0:35281:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.769895:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a6:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.769897:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.769902:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a6:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.769904:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025465 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.769910:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc02548f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.769918:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025465 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.769934:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025465 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.769938:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514f9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+140:3628:x1870152993084416:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3756us (3831us total) trans 4294975705 rc 0/0 00010000:00010000:3.0:1783520935.769940:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006b7bd685/0xd5401cbabc01f88c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00be expref: 858 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.769947:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025465 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.769959:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44205 00010000:00010000:3.0:1783520935.769968:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006b7bd685/0xd5401cbabc01f88c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00be expref: 858 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.769978:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.769982:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.769988:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.769991:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f3b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993086336:192.168.204.33@tcp:104: 00000004:00080000:1.0:1783520935.769999:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cc7f100/2, rc 0, transno 4294975705 00000100:00100000:1.0:1783520935.770013:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cc7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993084416:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.770017:0:35281:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: mdt-lustre-MDT0000_UUID lock: ffff943512185200/0xd5401cbabc025496 lrc: 3/0,1 mode: --/PW res: [0x200000402:0x72a:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x40000000000000 pid: 35281 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.770026:0:35281:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943512185200/0xd5401cbabc025496 lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210000000000 pid: 35281 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.770041:0:35281:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943512185200/0xd5401cbabc025496 lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 35281 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.770052:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435129670c0 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.770074:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025449 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x59d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.770080:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025449 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x59d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.770086:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025449 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x59d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000001:00080000:2.0:1783520935.770087:0:35281:0:(tgt_lastrcvd.c:2117:tgt_txn_stop_cb()) More than one transaction 8589957893 00000100:00100000:1.0:1783520935.770092:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:3.0:1783520935.770095:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025449 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x59d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.770103:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517244000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993086464:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.770109:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000004:00010000:2.0:1783520935.770116:0:35281:0:(mdt_handler.c:4478:mdt_save_lock()) ### save lock request 00000000fd25a99f reply state 00000000b9297bed transno 8589957894 ns: mdt-lustre-MDT0000_UUID lock: ffff943512185200/0xd5401cbabc025496 lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 35281 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.770120:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.770128:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025449 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x59d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.770135:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.770143:0:35281:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943512185200/0xd5401cbabc025496 lrc: 3/0,1 mode: TXN/TXN res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210000000000 pid: 35281 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.770144:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.770151:0:35281:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x72a:0x0].0x0 (ffff943515473e00) 00010000:00010000:0.0:1783520935.770152:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025449 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x59d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.770154:0:35281:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00000100:00100000:2.0:1783520935.770157:0:35281:0:(service.c:179:ptlrpc_save_lock()) nlocks 0 00010000:00010000:0.0:1783520935.770166:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025449 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x59d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.770167:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993086464 00000100:00100000:1.0:1783520935.770173:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993086464 00010000:00010000:0.0:1783520935.770184:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025449 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x59d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.770193:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025449 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x59d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.770202:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44210 00000100:00100000:2.0:1783520935.770207:0:35281:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c551500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_012:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+25:251795:x1870153002380672:12345-192.168.204.33@tcp:36:dd.0 Request processed in 420us (571us total) trans 8589957894 rc 0/0 00000100:00100000:1.0:1783520935.770214:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635600450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+140:3626:x1870152993086464:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.770216:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.770218:0:35281:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98404 00010000:00010000:0.0:1783520935.770220:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.770226:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00080000:2.0:1783520935.770227:0:35281:0:(tgt_main.c:784:tgt_ses_key_exit()) total 2 transactions per RPC 00002000:00080000:1.0:1783520935.770233:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1447 count 1 00010000:00010000:1.0:1783520935.770247:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc02549d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5a7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.770260:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006274216c/0xd5401cbabc01ff62 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f00c5 expref: 858 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.770267:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a7:0x0].0x0 since=37s v=(0 0) 00000004:00080000:2.0:1783520935.770268:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517244700/2, rc 0, transno 4294975703 00010000:00010000:1.0:1783520935.770272:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.770278:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc02549d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.770279:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517244700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993084288:0@lo:6:osp-syn-1-0.0 00000004:00080000:2.0:1783520935.770293:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517245500/2, rc 0, transno 4294975704 00000100:00100000:3.0:1783520935.770300:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cc7ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+141:3627:x1870152993083904:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4511us (4580us total) trans 4294975706 rc 0/0 00000100:00100000:2.0:1783520935.770301:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517245500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993084160:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.770306:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006274216c/0xd5401cbabc01ff62 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00c5 expref: 858 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.770317:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44201 00010000:00010000:1.0:1783520935.770324:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006274216c/0xd5401cbabc01ff62 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00c5 expref: 858 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.770337:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c550e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993086592:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.770406:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6300 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.770414:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943505875500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993086720:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.770424:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b6e760f4/0xd5401cbabc01fd1d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00a9 expref: 858 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.770426:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943505874700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993086848:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.770443:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993085696:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.770447:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895000 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.770460:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.770462:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02547a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.770478:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02547a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.770486:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02547a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.770488:0:20415:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e9610a18/0xd5401cbabc01f885 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00b0 expref: 857 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.770514:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02547a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.770517:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993086720 00000100:00100000:2.0:1783520935.770522:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943638996680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993085824:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.770524:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993086720 00010000:00010000:3.0:1783520935.770533:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02547a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.770540:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.770541:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025481 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.770552:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993086848 00000100:00100000:0.0:1783520935.770558:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993086848 00010000:00010000:2.0:1783520935.770560:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025481 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.770569:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000004:00080000:1.0:1783520935.770569:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e9e300/2, rc 0, transno 4294975706 00010000:00010000:2.0:1783520935.770576:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025481 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.770583:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993083904:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.770584:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025481 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.770589:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44211 00010000:00010000:2.0:1783520935.770598:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025481 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.770603:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517247800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+142:3626:x1870152993086720:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.770610:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.770616:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02547a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.770619:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1448 count 1 00000100:00100000:1.0:1783520935.770623:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c551f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993086976:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.770624:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.770629:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02547a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.770636:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02547a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.770636:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc0254a4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5a8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.770648:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.770651:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02547a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.770660:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02547a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.770662:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000058b492c7/0xd5401cbabc01f8d2 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f00cc expref: 856 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.770682:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993086976 00010000:00010000:0.0:1783520935.770684:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a8:0x0].0x0 since=37s v=(0 0) 00000100:00100000:1.0:1783520935.770688:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993086976 00010000:00010000:0.0:1783520935.770688:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.770693:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.770694:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc0254a4 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.770696:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.770698:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.770715:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000058b492c7/0xd5401cbabc01f8d2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00cc expref: 856 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.770716:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.770719:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44212 00000100:00100000:1.0:1783520935.770728:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c553100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+144:3627:x1870152993086848:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.770732:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000058b492c7/0xd5401cbabc01f8d2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00cc expref: 856 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.770740:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1449 count 1 00000100:00100000:0.0:1783520935.770747:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517244e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993087104:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.770751:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0254ab lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5a9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.770762:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001d351f1d/0xd5401cbabc01fd2b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f00d3 expref: 856 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.770767:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a9:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.770769:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.770772:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0254ab lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.770783:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638996d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+145:3625:x1870152993085440:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3435us (3634us total) trans 4294975707 rc 0/0 00000100:00100000:0.0:1783520935.770785:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470f00 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.770792:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001d351f1d/0xd5401cbabc01fd2b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00d3 expref: 856 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.770798:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44206 00010000:00010000:0.0:1783520935.770801:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.770809:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025481 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.770811:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c552300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+145:3626:x1870152993085568:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3335us (3396us total) trans 4294975708 rc 0/0 00010000:00010000:1.0:1783520935.770813:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001d351f1d/0xd5401cbabc01fd2b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00d3 expref: 856 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.770821:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44213 00000100:00100000:2.0:1783520935.770832:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44207 00010000:00010000:0.0:1783520935.770832:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:3.0:1783520935.770835:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c552d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+144:3628:x1870152993086976:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.770841:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025481 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.770846:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c552a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993087232:192.168.204.33@tcp:104: 00002000:00080000:3.0:1783520935.770851:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1450 count 1 00010000:00010000:0.0:1783520935.770858:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025481 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.770868:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0254b2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5aa:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520935.770873:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517247b80/2, rc 0, transno 4294975707 00010000:00010000:0.0:1783520935.770879:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025481 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.770880:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6f00 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.770885:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517247b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993085440:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.770896:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025481 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.770909:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.770911:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.770914:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.770933:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c552680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993087360:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.770952:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d07b3a18/0xd5401cbabc022897 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f87 expref: 856 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.770957:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943638994e00/2, rc 0, transno 4294975708 00000100:00100000:2.0:1783520935.770967:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.770974:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943638994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993085568:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.770977:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x5aa:0x0].0x0 since=36s v=(0 0) 00010000:00010000:3.0:1783520935.770982:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5aa:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520935.770987:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0254b2 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.771001:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993087360 00000100:00100000:2.0:1783520935.771004:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993087360 00010000:00010000:3.0:1783520935.771013:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d07b3a18/0xd5401cbabc022897 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f87 expref: 856 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.771024:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d07b3a18/0xd5401cbabc022897 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f87 expref: 856 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.771030:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517245c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993087488:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.771035:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44214 00000100:00100000:3.0:1783520935.771041:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993087616:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.771049:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505876300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+144:3625:x1870152993087360:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.771065:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1451 count 1 00000100:00100000:1.0:1783520935.771067:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.771081:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0254b9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5ab:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.771092:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993087488 00010000:00010000:2.0:1783520935.771094:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000026287415/0xd5401cbabc01fd32 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f00e1 expref: 856 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.771096:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993087488 00010000:00010000:2.0:1783520935.771106:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ab:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.771110:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ab:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.771113:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0254b9 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.771120:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44215 00000100:00100000:0.0:1783520935.771127:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c553480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+145:3626:x1870152993087488:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.771132:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000026287415/0xd5401cbabc01fd32 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00e1 expref: 856 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.771139:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1452 count 1 00010000:00010000:2.0:1783520935.771140:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000026287415/0xd5401cbabc01fd32 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00e1 expref: 856 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.771151:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350794ca80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993087744:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.771153:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0254c0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5ac:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.771164:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b05b05dd/0xd5401cbabc01f89a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f00e8 expref: 856 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.771169:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ac:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.771171:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ac:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.771174:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0254c0 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.771187:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895600 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.771188:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967840 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.771198:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b05b05dd/0xd5401cbabc01f89a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00e8 expref: 856 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.771209:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b05b05dd/0xd5401cbabc01f89a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00e8 expref: 856 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.771222:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad1180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993087872:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.771226:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f3b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993086336:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.771247:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc02548f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.771254:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124706c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.771279:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c550e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993086592:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.771297:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc02549d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.771650:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.771695:0:178509:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002380928 00000100:00100000:0.0:1783520935.771695:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.771700:0:178509:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002380928 00002000:00010000:1.0:1783520935.771725:0:178509:0:(ofd_dev.c:2825:ofd_punch_hpreq_check()) lustre-OST0001: refresh locks: 11811161089/2295 (196608->18446744073709551615) 00000100:00100000:3.0:1783520935.771729:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002380800 00002000:00100000:1.0:1783520935.771730:0:178509:0:(ofd_dev.c:2565:ofd_prolong_extent_locks()) Prolong locks for req ffff943507ad1880 with x1870153002380928 ext(196608->18446744073709551615) 00000100:00100000:2.0:1783520935.771731:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943505875f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993086208:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.771737:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002380800 00000100:00100000:0.0:1783520935.771740:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00002000:00010000:1.0:1783520935.771742:0:178509:0:(ofd_dev.c:2830:ofd_punch_hpreq_check()) lustre-OST0001: refreshed 0 locks timeout for req ffff943507ad1880. 00010000:00010000:2.0:1783520935.771760:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025488 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.771764:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006b7bd685/0xd5401cbabc01f88c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00be expref: 858 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.771772:0:178509:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 34735 00000100:00100000:1.0:1783520935.771784:0:178509:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_io00_014:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+858:2387:x1870153002380928:12345-192.168.204.33@tcp:10:dd.0 00000100:00100000:3.0:1783520935.771806:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35801 00000100:00100000:2.0:1783520935.771807:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002381056 00000100:00100000:2.0:1783520935.771813:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002381056 00010000:00010000:2.0:1783520935.771835:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006274216c/0xd5401cbabc01ff62 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00c5 expref: 859 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.771863:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35802 00000100:00100000:2.0:1783520935.771873:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad0700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+860:2387:x1870153002381056:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520935.771879:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+860:2387:x1870153002380800:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.771883:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.771891:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006274216c/0xd5401cbabc01ff62 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00c5 expref: 860 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.771892:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00002000:00010000:1.0:1783520935.771896:0:178509:0:(ofd_lvb.c:306:ofd_lvbo_update()) res: [0x2c0000401:0x8f7:0x0] updating lvb size from disk: 65536 -> 196608 00010000:00010000:3.0:1783520935.771903:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006b7bd685/0xd5401cbabc01f88c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00be expref: 860 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00010000:1.0:1783520935.771903:0:178509:0:(ofd_lvb.c:313:ofd_lvbo_update()) res: [0x2c0000401:0x8f7:0x0] updating lvb mtime from disk: 1783520935 -> 1783520936 00002000:00010000:1.0:1783520935.771910:0:178509:0:(ofd_lvb.c:319:ofd_lvbo_update()) res: [0x2c0000401:0x8f7:0x0] updating lvb atime from disk: 0 -> 0 00002000:00010000:1.0:1783520935.771915:0:178509:0:(ofd_lvb.c:325:ofd_lvbo_update()) res: [0x2c0000401:0x8f7:0x0] updating lvb ctime from disk: 1783520935 -> 1783520936 00002000:00010000:1.0:1783520935.771920:0:178509:0:(ofd_lvb.c:331:ofd_lvbo_update()) res: [0x2c0000401:0x8f7:0x0] updating lvb blocks from disk: 0 -> 0 00010000:00010000:3.0:1783520935.771936:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006b7bd685/0xd5401cbabc01f88c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5a6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00be expref: 860 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.771939:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006274216c/0xd5401cbabc01ff62 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5a7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00c5 expref: 860 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.771952:0:178509:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_io00_014:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+860:2387:x1870153002380928:12345-192.168.204.33@tcp:10:dd.0 Request processed in 169us (301us total) trans 4294975709 rc 0/0 00010000:00010000:3.0:1783520935.771964:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006b7bd685/0xd5401cbabc01f88c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5a6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00be expref: 860 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.771967:0:178509:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 34735 00002000:00010000:1.0:1783520935.771973:0:178509:0:(ofd_dev.c:2825:ofd_punch_hpreq_check()) lustre-OST0001: refresh locks: 11811161089/2295 (196608->18446744073709551615) 00010000:00010000:2.0:1783520935.771974:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006274216c/0xd5401cbabc01ff62 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5a7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00c5 expref: 860 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00100000:1.0:1783520935.771981:0:178509:0:(ofd_dev.c:2565:ofd_prolong_extent_locks()) Prolong locks for req ffff943507ad1880 with x1870153002380928 ext(196608->18446744073709551615) 00010000:00010000:3.0:1783520935.771986:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a6:0x0].0x0 since=0s v=(0 0) 00002000:00010000:1.0:1783520935.771990:0:178509:0:(ofd_dev.c:2830:ofd_punch_hpreq_check()) lustre-OST0001: refreshed 0 locks timeout for req ffff943507ad1880. 00010000:00010000:3.0:1783520935.771995:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a6:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.772001:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc02548f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.772004:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.772013:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.772016:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc02548f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.772019:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc02549d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.772035:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc02549d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.772047:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.772056:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc02548f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.772058:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.772069:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc02548f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.772071:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc02549d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.772082:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+857:2387:x1870153002380800:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 264us (384us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.772089:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad0700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+857:2387:x1870153002381056:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 214us (351us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.772091:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc02548f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.772093:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc02549d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.772103:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35801 00000100:00100000:2.0:1783520935.772118:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35802 00010000:00010000:1.0:1783520935.772119:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc02549d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.772120:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc02548f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.772138:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc02549d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.772149:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520935.772157:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:1.0:1783520935.772245:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635600450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+147:3626:x1870152993086464:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2034us (2113us total) trans 4294975710 rc 0/0 00000100:00100000:1.0:1783520935.772253:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44210 00000004:00080000:2.0:1783520935.772265:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517244000/2, rc 0, transno 4294975710 00000100:00100000:0.0:1783520935.772275:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c551880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+146:3627:x1870152993086080:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2511us (2698us total) trans 4294975711 rc 0/0 00000100:00100000:2.0:1783520935.772278:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517244000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993086464:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.772290:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44209 00000100:00100000:2.0:1783520935.772314:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:0.0:1783520935.772315:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520935.772323:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc02548f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.772329:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c550e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993088000:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.772335:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5c00 already unused, calling callback (ffffffffc0bf9e90) 00000004:00080000:3.0:1783520935.772337:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c553b80/2, rc 0, transno 4294975711 00010000:00010000:0.0:1783520935.772340:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc02548f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.772352:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c553b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993086080:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.772352:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc02548f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.772386:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.772390:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc02548f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.772400:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc02548f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.772422:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c550a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993088128:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.772423:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.772429:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc02549d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.772437:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.772441:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc02549d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.772455:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc02549d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.772461:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.772473:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc02549d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.772481:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc02549d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.772507:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517244e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993087104:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.772509:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.772512:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.772516:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.772519:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.772526:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.772527:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc0254a4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.772535:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993088000 00000100:00100000:0.0:1783520935.772540:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993088000 00000100:00100000:1.0:1783520935.772552:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993088128 00000100:00100000:1.0:1783520935.772557:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993088128 00000100:00100000:0.0:1783520935.772573:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44216 00000100:00100000:0.0:1783520935.772584:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350794f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+147:3628:x1870152993088000:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.772597:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44217 00002000:00080000:0.0:1783520935.772599:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1453 count 1 00000100:00100000:2.0:1783520935.772611:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350794dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+147:3628:x1870152993088128:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.772616:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0254c7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5ad:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.772625:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00002000:00080000:2.0:1783520935.772628:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1454 count 1 00000100:00100000:1.0:1783520935.772630:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c552a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993087232:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.772640:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005f6f5de1/0xd5401cbabc01fd47 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f00ef expref: 855 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.772647:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0254ce lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5ae:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.772649:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ad:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.772652:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0254ab lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.772658:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ad:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.772670:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0254c7 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.772677:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000019d7118a/0xd5401cbabc01f8a1 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f00f6 expref: 855 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.772682:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.772693:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ae:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.772697:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ae:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.772701:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0254ce lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.772703:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005f6f5de1/0xd5401cbabc01fd47 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00ef expref: 855 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.772711:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002381184 00000100:00100000:1.0:1783520935.772718:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002381184 00010000:00010000:0.0:1783520935.772728:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005f6f5de1/0xd5401cbabc01fd47 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00ef expref: 856 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.772737:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000019d7118a/0xd5401cbabc01f8a1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00f6 expref: 856 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.772741:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007c33b957/0xd5401cbabc01fd24 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00b7 expref: 856 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.772743:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993088256:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.772767:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000019d7118a/0xd5401cbabc01f8a1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00f6 expref: 856 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.772774:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002381312 00000100:00100000:1.0:1783520935.772777:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002381312 00000100:00100000:2.0:1783520935.772781:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350794d500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993088384:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.772792:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000058b492c7/0xd5401cbabc01f8d2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00cc expref: 857 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.772813:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002381440 00000100:00100000:1.0:1783520935.772816:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002381440 00010000:00010000:1.0:1783520935.772831:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001d351f1d/0xd5401cbabc01fd2b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00d3 expref: 858 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.772851:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895000 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.772851:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470180 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.772853:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35803 00000100:00100000:1.0:1783520935.772863:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435079f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+858:2387:x1870153002381184:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.772874:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.772879:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007c33b957/0xd5401cbabc01fd24 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00b7 expref: 858 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.772883:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35804 00000100:00100000:2.0:1783520935.772888:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350794ca80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993087744:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.772896:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007c33b957/0xd5401cbabc01fd24 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5a5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00b7 expref: 858 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.772902:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.772903:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435079f0000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+858:2388:x1870153002381312:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.772908:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0254b9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.772917:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007c33b957/0xd5401cbabc01fd24 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5a5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00b7 expref: 858 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.772919:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.772929:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000058b492c7/0xd5401cbabc01f8d2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00cc expref: 857 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.772930:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.772935:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.772942:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025488 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.772948:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025488 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.772951:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000058b492c7/0xd5401cbabc01f8d2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5a8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00cc expref: 857 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.772958:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.772978:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.772979:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000058b492c7/0xd5401cbabc01f8d2 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5a8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00cc expref: 857 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.772988:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025488 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.772993:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.773001:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025488 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.773001:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a8:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.773002:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435079f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+856:2387:x1870153002381184:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 137us (505us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.773013:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc0254a4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.773022:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35803 00000100:00100000:3.0:1783520935.773024:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.773027:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025488 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.773036:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc0254a4 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.773042:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025488 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.773045:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002381568 00000100:00100000:1.0:1783520935.773049:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002381568 00010000:00010000:0.0:1783520935.773061:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.773065:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.773065:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d07b3a18/0xd5401cbabc022897 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x5aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f87 expref: 856 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.773079:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.773085:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025488 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.773086:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002381696 00000100:00100000:1.0:1783520935.773090:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002381696 00000100:00100000:0.0:1783520935.773091:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435079f0000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+856:2388:x1870153002381312:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 194us (468us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.773102:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000026287415/0xd5401cbabc01fd32 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00e1 expref: 857 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.773105:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520935.773109:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35804 00010000:00010000:2.0:1783520935.773112:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025488 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.773119:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002381824 00010000:00010000:2.0:1783520935.773123:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025488 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.773126:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002381824 00000100:00100000:1.0:1783520935.773127:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35805 00000100:00100000:0.0:1783520935.773136:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35806 00000100:00100000:1.0:1783520935.773141:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357c450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+857:2387:x1870153002381440:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.773144:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025488 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.773149:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.773152:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435079f1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+857:2387:x1870153002381568:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.773153:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b05b05dd/0xd5401cbabc01f89a lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00e8 expref: 857 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.773157:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001d351f1d/0xd5401cbabc01fd2b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5a9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00d3 expref: 857 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.773167:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025488 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.773170:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.773171:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001d351f1d/0xd5401cbabc01fd2b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5a9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00d3 expref: 857 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.773181:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d07b3a18/0xd5401cbabc022897 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x5aa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f87 expref: 857 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.773201:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.773201:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001d351f1d/0xd5401cbabc01fd2b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5a9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00d3 expref: 857 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.773203:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d07b3a18/0xd5401cbabc022897 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5aa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f87 expref: 857 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.773205:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.773210:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.773225:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35807 00010000:00010000:1.0:1783520935.773231:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.773233:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5aa:0x0].0x0 since=0s v=(1 0) 00000100:00100000:3.0:1783520935.773239:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435079f1500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+856:2387:x1870153002381696:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.773240:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5a9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.773241:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5aa:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.773250:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0254ab lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.773253:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0254b2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5aa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.773256:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.773267:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000026287415/0xd5401cbabc01fd32 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ab:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00e1 expref: 856 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.773269:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0254ab lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.773284:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0254b2 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5aa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.773296:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c553800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+148:3627:x1870152993085952:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3816us (4441us total) trans 4294975712 rc 0/0 00010000:00010000:3.0:1783520935.773298:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000026287415/0xd5401cbabc01fd32 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5ab:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00e1 expref: 856 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.773316:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44208 00010000:00010000:1.0:1783520935.773331:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.773336:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.773337:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000026287415/0xd5401cbabc01fd32 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5ab:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00e1 expref: 856 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.773352:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ab:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.773380:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357c450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+855:2387:x1870153002381440:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 219us (681us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.773388:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ab:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.773388:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35805 00010000:00010000:3.0:1783520935.773395:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0254b9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.773398:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435079f1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+854:2387:x1870153002381568:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 250us (503us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.773412:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35806 00000100:00100000:1.0:1783520935.773439:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35808 00000100:00100000:1.0:1783520935.773450:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435079f2d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+853:2387:x1870153002381824:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.773460:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.773467:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b05b05dd/0xd5401cbabc01f89a lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5ac:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00e8 expref: 853 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.773487:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0254b9 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.773494:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b05b05dd/0xd5401cbabc01f89a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ac:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00e8 expref: 853 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.773521:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ac:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.773524:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ac:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.773526:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.773529:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0254c0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ac:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.773537:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0254b9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.773539:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0254c0 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5ac:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.773560:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435079f1500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+853:2387:x1870153002381696:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 324us (605us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.773563:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0254b9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.773567:0:8408:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b05b05dd/0xd5401cbabc01f89a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5ac:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00e8 expref: 853 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.773570:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.773579:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35807 00010000:00010000:2.0:1783520935.773590:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0254b9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.773595:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad1180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993087872:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.773597:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435079f2d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+852:2387:x1870153002381824:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 147us (576us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.773612:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0254b9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.773613:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0254c0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.773620:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35808 00010000:00010000:0.0:1783520935.773636:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0254c0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.773641:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.773644:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0254c0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.773655:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0254c0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.773662:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0254c0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.773665:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c550380/2, rc 0, transno 4294975712 00000100:00100000:1.0:1783520935.773680:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c550380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993085952:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.773685:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:2.0:1783520935.773732:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505876300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+149:3625:x1870152993087360:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2686us (2765us total) trans 4294975713 rc 0/0 00000100:00100000:2.0:1783520935.773742:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44214 00010000:00010000:3.0:1783520935.773743:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d07b3a18/0xd5401cbabc022897 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f87 expref: 850 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.773743:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc0254a4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.773770:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc0254a4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.773775:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993087616:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.773778:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c553480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+148:3626:x1870152993087488:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2647us (2708us total) trans 4294975714 rc 0/0 00000004:00080000:2.0:1783520935.773781:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517245c00/2, rc 0, transno 4294975714 00010000:00010000:1.0:1783520935.773792:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc0254a4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.773794:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44215 00000100:00100000:2.0:1783520935.773798:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517245c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993087488:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.773810:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc0254a4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.773811:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:0.0:1783520935.773815:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520935.773822:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0254b9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.773824:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c552a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993088512:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.773829:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520935.773837:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0254b2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.773838:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.773845:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0254b9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.773853:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0254b2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.773857:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0254b9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.773866:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.773877:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0254b2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.773884:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0254b9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.773887:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0254b2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.773899:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0254b9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.773903:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0254b2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520935.773914:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c552680/2, rc 0, transno 4294975713 00000100:00100000:2.0:1783520935.773922:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c552680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993087360:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.773923:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00000100:00100000:1.0:1783520935.773924:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517247800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+148:3626:x1870152993086720:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3321us (3463us total) trans 4294975715 rc 0/0 00010000:00010000:0.0:1783520935.773931:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0254c0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.773938:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44211 00010000:00010000:0.0:1783520935.773944:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfa600 already unused, calling callback (ffffffffc0bf9e90) 00000004:00080000:2.0:1783520935.773948:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943505875500/2, rc 0, transno 4294975715 00010000:00010000:0.0:1783520935.773950:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0254c0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.773957:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943505875500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993086720:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.773960:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993088512 00000100:00100000:1.0:1783520935.773967:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993088512 00010000:00010000:0.0:1783520935.773985:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0254c0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.773991:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:0.0:1783520935.773998:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0254c0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.774002:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44218 00010000:00010000:2.0:1783520935.774003:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520935.774011:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc0254a4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.774013:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0254c0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.774018:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350794ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+148:3626:x1870152993088512:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.774036:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb600 already unused, calling callback (ffffffffc0bf9e90) 00002000:00080000:1.0:1783520935.774040:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1455 count 1 00010000:00010000:2.0:1783520935.774041:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc0254a4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.774045:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.774052:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0254b2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.774059:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc0254a4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.774060:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0254d5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5af:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.774061:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.774069:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0254b2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.774084:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c552d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+149:3628:x1870152993086976:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3249us (3435us total) trans 4294975716 rc 0/0 00010000:00010000:2.0:1783520935.774085:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc0254a4 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.774091:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0254b2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.774092:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000071affd4c/0xd5401cbabc01fd39 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f00fd expref: 849 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.774109:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc0254a4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.774119:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5af:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.774123:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0254b2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.774129:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5af:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.774140:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0254d5 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.774142:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0254b2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.774148:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.774152:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.774156:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.774165:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44213 00010000:00010000:1.0:1783520935.774178:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000071affd4c/0xd5401cbabc01fd39 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00fd expref: 849 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.774185:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.774189:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.774191:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000071affd4c/0xd5401cbabc01fd39 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00fd expref: 849 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.774193:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520935.774202:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c551f80/2, rc 0, transno 4294975716 00010000:00010000:0.0:1783520935.774204:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.774207:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c550000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993088640:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.774211:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c551f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993086976:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.774212:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.774261:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6f00 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.774300:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993088256:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.774321:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0254c7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.774322:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c551180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993088768:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.774352:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c551880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993089152:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.774353:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.774376:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c550700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993088896:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.774427:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c552300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993089024:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.774447:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.774473:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.774475:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993088768 00000100:00100000:0.0:1783520935.774481:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993088768 00000100:00100000:2.0:1783520935.774525:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350794d500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993088384:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.774525:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993089152 00000100:00100000:0.0:1783520935.774531:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993089152 00000100:00100000:1.0:1783520935.774535:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.774548:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0254ce lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.774564:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993088896 00000100:00100000:0.0:1783520935.774568:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993088896 00000100:00100000:2.0:1783520935.774574:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993089024 00000100:00100000:2.0:1783520935.774579:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993089024 00000100:00100000:1.0:1783520935.774585:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44219 00000100:00100000:0.0:1783520935.774592:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44220 00000100:00100000:1.0:1783520935.774597:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435079f0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+152:3626:x1870152993088768:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.774605:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350794d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+152:3625:x1870152993089152:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.774610:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44221 00002000:00080000:1.0:1783520935.774613:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1456 count 1 00002000:00080000:0.0:1783520935.774623:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1459 count 1 00000100:00100000:2.0:1783520935.774624:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c553800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+152:3627:x1870152993088896:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.774628:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0254dc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5b0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.774629:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44222 00010000:00010000:0.0:1783520935.774640:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0254e3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5b3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.774645:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006ac6c1ba/0xd5401cbabc01f8a8 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0104 expref: 849 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.774646:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1457 count 1 00000100:00100000:3.0:1783520935.774647:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517244e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+152:3628:x1870152993089024:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.774664:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b0:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.774665:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0254ea lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5b1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.774669:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000003929aa7/0xd5401cbabc01fd4e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0119 expref: 849 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520935.774671:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1458 count 1 00010000:00010000:1.0:1783520935.774671:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.774682:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0254dc lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.774690:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0254f1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5b2:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.774693:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b3:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.774698:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007a91c72a/0xd5401cbabc01fd40 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f010b expref: 849 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.774702:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.774712:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0254e3 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.774724:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006ac6c1ba/0xd5401cbabc01f8a8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0104 expref: 849 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.774737:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b1:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.774743:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006ac6c1ba/0xd5401cbabc01f8a8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0104 expref: 849 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.774744:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.774750:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000003929aa7/0xd5401cbabc01fd4e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0119 expref: 849 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.774755:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0254ea lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.774772:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005927358f/0xd5401cbabc01f8af lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0112 expref: 849 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.774773:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993089280:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.774795:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b2:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.774800:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000003929aa7/0xd5401cbabc01fd4e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0119 expref: 849 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.774801:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.774802:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007a91c72a/0xd5401cbabc01fd40 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f010b expref: 849 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.774812:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0254f1 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.774820:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6300 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.774832:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007a91c72a/0xd5401cbabc01fd40 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f010b expref: 849 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.774833:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad1180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993089408:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.774841:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0254ab lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.774848:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005927358f/0xd5401cbabc01f8af lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0112 expref: 849 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.774860:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350794d500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993089536:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.774868:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124706c0 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.774869:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0254ab lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.774893:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0254ab lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.774895:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005927358f/0xd5401cbabc01f8af lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0112 expref: 849 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.774898:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895000 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.774911:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.774918:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0254ab lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.774925:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f3800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993089664:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.774942:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520935.774943:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002382080 00000100:00100000:2.0:1783520935.774949:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002382080 00000100:00100000:3.0:1783520935.774964:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435129670c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.774970:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000019d7118a/0xd5401cbabc01f8a1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00f6 expref: 850 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.774990:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.774996:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35809 00000100:00100000:2.0:1783520935.775008:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad1c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+850:2388:x1870153002382080:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.775018:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.775021:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002381952 00010000:00010000:2.0:1783520935.775024:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000019d7118a/0xd5401cbabc01f8a1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ae:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00f6 expref: 850 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.775026:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002381952 00000100:00100000:1.0:1783520935.775048:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c553100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+153:3627:x1870152993086848:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4317us (4507us total) trans 4294975717 rc 0/0 00010000:00010000:2.0:1783520935.775050:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000019d7118a/0xd5401cbabc01f8a1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5ae:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00f6 expref: 851 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.775050:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005f6f5de1/0xd5401cbabc01fd47 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00ef expref: 851 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.775059:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44212 00010000:00010000:2.0:1783520935.775081:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000019d7118a/0xd5401cbabc01f8a1 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5ae:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00f6 expref: 851 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.775089:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35810 00010000:00010000:2.0:1783520935.775093:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ae:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.775097:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ae:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.775098:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+850:2388:x1870153002381952:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000004:00080000:1.0:1783520935.775099:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943505874700/2, rc 0, transno 4294975717 00010000:00010000:2.0:1783520935.775105:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0254ce lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.775110:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.775111:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0254ce lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.775115:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943505874700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993086848:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.775117:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005f6f5de1/0xd5401cbabc01fd47 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ad:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00ef expref: 850 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.775133:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005f6f5de1/0xd5401cbabc01fd47 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5ad:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00ef expref: 850 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.775137:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.775146:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005f6f5de1/0xd5401cbabc01fd47 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5ad:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00ef expref: 850 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.775160:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ad:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.775164:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ad:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.775168:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0254c7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.775174:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0254c7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.775181:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993089792:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.775182:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad1c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+849:2388:x1870153002382080:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 175us (275us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.775188:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.775197:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35809 00000100:00100000:0.0:1783520935.775211:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+848:2388:x1870153002381952:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 115us (223us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.775214:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520935.775219:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35810 00010000:00010000:2.0:1783520935.775220:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0254ab lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.775226:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.775234:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.775239:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0254ab lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.775253:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0254ab lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.775259:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993089792 00000100:00100000:0.0:1783520935.775264:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993089792 00010000:00010000:2.0:1783520935.775267:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0254ab lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.775276:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0254ab lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.775288:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c550000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993088640:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.775293:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.775297:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.775297:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44223 00010000:00010000:2.0:1783520935.775301:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.775305:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0254d5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.775310:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943611038e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+153:3626:x1870152993089792:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.775311:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0254ce lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.775323:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0254ce lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.775329:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0254ce lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.775330:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1460 count 1 00010000:00010000:2.0:1783520935.775340:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0254ce lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.775345:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0254f8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5b4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.775403:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.775409:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.775417:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0254ce lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.775418:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009195ead4/0xd5401cbabc01f8b6 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0120 expref: 847 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.775441:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.775441:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b4:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.775447:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.775448:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0254ce lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.775455:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0254f8 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.775458:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0254ce lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.775476:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0254ce lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.775483:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009195ead4/0xd5401cbabc01f8b6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0120 expref: 847 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.775484:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0254ce lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.775508:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350794dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+154:3628:x1870152993088128:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2885us (3034us total) trans 4294975718 rc 0/0 00010000:00010000:3.0:1783520935.775516:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0254c7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.775520:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009195ead4/0xd5401cbabc01f8b6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0120 expref: 847 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.775522:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44217 00010000:00010000:1.0:1783520935.775525:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.775529:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.775533:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.775539:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0254c7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.775546:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993089920:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.775552:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0254c7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520935.775558:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c550a80/2, rc 0, transno 4294975718 00010000:00010000:3.0:1783520935.775566:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0254c7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.775574:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c550a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993088128:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.775598:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.775604:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0254c7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.775613:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfbc00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520935.775614:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943611038700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993090048:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.775618:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0254c7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.775629:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0254c7 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.775642:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0254c7 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x5ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.775644:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.775648:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.775655:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0254c7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.775678:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993090048 00010000:00010000:0.0:1783520935.775678:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.775681:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.775684:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993090048 00010000:00010000:0.0:1783520935.775685:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.775711:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.775719:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44224 00000100:00100000:2.0:1783520935.775730:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943611039500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+155:3627:x1870152993090048:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.775736:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002382208 00000100:00100000:1.0:1783520935.775740:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002382208 00002000:00080000:2.0:1783520935.775746:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1461 count 1 00000100:00100000:3.0:1783520935.775747:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350794f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+155:3628:x1870152993088000:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3164us (3359us total) trans 4294975719 rc 0/0 00010000:00010000:1.0:1783520935.775759:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000071affd4c/0xd5401cbabc01fd39 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00fd expref: 848 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.775760:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44216 00010000:00010000:2.0:1783520935.775764:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0254ff lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5b5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.775788:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b23a7b29/0xd5401cbabc01fd55 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0127 expref: 848 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.775796:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b5:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.775800:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b5:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.775803:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.775805:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0254ff lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.775807:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35811 00000100:00100000:1.0:1783520935.775819:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+848:2388:x1870153002382208:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.775831:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b23a7b29/0xd5401cbabc01fd55 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0127 expref: 848 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.775831:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.775839:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000071affd4c/0xd5401cbabc01fd39 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5af:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00fd expref: 848 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.775848:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b23a7b29/0xd5401cbabc01fd55 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0127 expref: 848 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.775860:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000071affd4c/0xd5401cbabc01fd39 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5af:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00fd expref: 848 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.775885:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350794e300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993090176:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.775898:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000071affd4c/0xd5401cbabc01fd39 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5af:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f00fd expref: 848 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.775907:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5af:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.775910:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5af:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.775913:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0254d5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.775918:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0254d5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.775922:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895840 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.775941:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000004:00080000:2.0:1783520935.775954:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c550e00/2, rc 0, transno 4294975719 00010000:00010000:0.0:1783520935.775959:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0254d5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.775970:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c550e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993088000:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.775972:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+847:2388:x1870153002382208:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 151us (256us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.775973:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0254d5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.775989:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35811 00010000:00010000:0.0:1783520935.775992:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0254d5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.776003:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0254d5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.776020:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.776023:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1783520935.776033:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350794ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993090304:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.776064:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.776094:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993090304 00000100:00100000:2.0:1783520935.776099:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993090304 00000100:00100000:0.0:1783520935.776103:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350794ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+155:3626:x1870152993088512:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2089us (2240us total) trans 4294975720 rc 0/0 00000100:00100000:0.0:1783520935.776112:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44218 00000100:00100000:1.0:1783520935.776121:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993089280:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.776139:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44225 00010000:00010000:1.0:1783520935.776140:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0254dc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.776152:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94361103b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+155:3627:x1870152993090304:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.776156:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.776163:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0254d5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.776165:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1462 count 1 00000004:00080000:1.0:1783520935.776175:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c552a00/2, rc 0, transno 4294975720 00010000:00010000:0.0:1783520935.776176:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.776181:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025506 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5b6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.776183:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0254d5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.776190:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c552a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993088512:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.776207:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0254d5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.776209:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000084912c38/0xd5401cbabc01f8bd lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f012e expref: 846 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.776229:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b6:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.776233:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b6:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.776233:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0254d5 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.776236:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:2.0:1783520935.776239:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025506 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.776248:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993090432:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.776257:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0254d5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.776277:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.776279:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000084912c38/0xd5401cbabc01f8bd lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f012e expref: 846 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.776281:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.776286:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.776289:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.776299:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000084912c38/0xd5401cbabc01f8bd lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f012e expref: 846 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.776312:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350794c000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993090560:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.776320:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993090432 00000100:00100000:3.0:1783520935.776326:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993090432 00000100:00100000:2.0:1783520935.776348:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895600 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.776379:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44226 00000100:00100000:3.0:1783520935.776388:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943611038a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+156:3626:x1870152993090432:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520935.776405:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1463 count 1 00010000:00010000:3.0:1783520935.776419:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02550d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5b7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.776432:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c6f4370a/0xd5401cbabc01fd5c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0135 expref: 846 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.776439:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b7:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.776442:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.776445:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02550d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.776463:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c6f4370a/0xd5401cbabc01fd5c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0135 expref: 846 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.776473:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c6f4370a/0xd5401cbabc01fd5c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0135 expref: 846 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.776536:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f3b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993090688:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.776576:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967e40 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.776872:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.776914:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002382336 00000100:00100000:1.0:1783520935.776918:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002382336 00010000:00010000:1.0:1783520935.776933:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006ac6c1ba/0xd5401cbabc01f8a8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0104 expref: 847 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.776936:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad1180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993089408:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.776947:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.776959:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0254e3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.776972:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350794d500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993089536:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.776991:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0254ea lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.777016:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002382464 00000100:00100000:1.0:1783520935.777022:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002382464 00000100:00100000:2.0:1783520935.777031:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35812 00010000:00010000:1.0:1783520935.777039:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000003929aa7/0xd5401cbabc01fd4e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0119 expref: 848 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.777050:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357c850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+848:2388:x1870153002382336:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.777061:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.777068:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35813 00010000:00010000:2.0:1783520935.777070:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006ac6c1ba/0xd5401cbabc01f8a8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0104 expref: 848 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.777072:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.777082:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435079f0000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+848:2388:x1870153002382464:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.777086:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002382592 00010000:00010000:2.0:1783520935.777091:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006ac6c1ba/0xd5401cbabc01f8a8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5b0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0104 expref: 848 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.777091:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.777093:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002382592 00010000:00010000:1.0:1783520935.777099:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000003929aa7/0xd5401cbabc01fd4e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0119 expref: 849 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.777113:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000003929aa7/0xd5401cbabc01fd4e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5b3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0119 expref: 849 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.777116:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006ac6c1ba/0xd5401cbabc01f8a8 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5b0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0104 expref: 849 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.777116:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007a91c72a/0xd5401cbabc01fd40 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f010b expref: 849 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.777129:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000003929aa7/0xd5401cbabc01fd4e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5b3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0119 expref: 849 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.777148:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.777149:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.777158:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.777159:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b0:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.777164:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35814 00010000:00010000:1.0:1783520935.777165:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0254e3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.777168:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0254dc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.777180:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f3800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993089664:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.777181:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435079f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+847:2390:x1870153002382592:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.777183:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0254e3 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.777205:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0254f1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.777209:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.777213:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.777216:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0254dc lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.777219:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007a91c72a/0xd5401cbabc01fd40 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f010b expref: 847 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.777243:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435079f0000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+847:2388:x1870153002382464:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 163us (297us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.777248:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007a91c72a/0xd5401cbabc01fd40 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5b1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f010b expref: 847 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.777256:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.777256:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35813 00010000:00010000:0.0:1783520935.777269:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007a91c72a/0xd5401cbabc01fd40 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5b1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f010b expref: 847 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.777283:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b1:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.777285:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357c850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+845:2388:x1870153002382336:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 243us (410us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.777290:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b1:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.777296:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35812 00010000:00010000:0.0:1783520935.777298:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0254ea lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.777308:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0254ea lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.777329:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.777336:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.777347:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0254dc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.777410:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002382720 00000100:00100000:1.0:1783520935.777416:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002382720 00010000:00010000:2.0:1783520935.777418:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0254dc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.777430:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0254dc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.777435:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435079f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+844:2390:x1870153002382592:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 180us (291us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.777436:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005927358f/0xd5401cbabc01f8af lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0112 expref: 845 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.777445:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0254dc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.777457:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35814 00010000:00010000:2.0:1783520935.777476:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.777483:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35815 00000100:00100000:1.0:1783520935.777493:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435079f3800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+844:2388:x1870153002382720:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.777516:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520935.777523:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.777525:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005927358f/0xd5401cbabc01f8af lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0112 expref: 844 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.777539:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005927358f/0xd5401cbabc01f8af lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5b2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0112 expref: 844 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.777554:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005927358f/0xd5401cbabc01f8af lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5b2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0112 expref: 844 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.777565:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.777569:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.777573:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0254f1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.777581:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0254f1 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.777600:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.777609:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0254e3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.777620:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0254e3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.777630:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0254e3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.777632:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435079f3800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+843:2388:x1870153002382720:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 139us (295us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.777633:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435079f0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+157:3626:x1870152993088768:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3037us (3277us total) trans 4294975721 rc 0/0 00000100:00100000:1.0:1783520935.777651:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35815 00000100:00100000:2.0:1783520935.777655:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44219 00010000:00010000:0.0:1783520935.777655:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0254e3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.777661:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.777672:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002382848 00000100:00100000:1.0:1783520935.777677:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002382848 00010000:00010000:0.0:1783520935.777682:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520935.777685:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520935.777691:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0254dc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.777696:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009195ead4/0xd5401cbabc01f8b6 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0120 expref: 843 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.777701:0:6496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002383104 00010000:00010000:2.0:1783520935.777708:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6200 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520935.777708:0:6496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002383104 00010000:00010000:2.0:1783520935.777715:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0254dc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.777726:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0254dc lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.777735:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35816 00010000:00010000:2.0:1783520935.777743:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0254dc lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.777752:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0254dc lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.777757:0:6496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 9625 00010000:00010000:3.0:1783520935.777765:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0254f1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.777773:0:6496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435079f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_rdpg00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+25:251795:x1870153002383104:12345-192.168.204.33@tcp:35:dd.0 00010000:00010000:2.0:1783520935.777775:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.777777:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.777783:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0254e3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.777793:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435079f1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+843:2388:x1870153002382848:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.777796:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfbe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.777802:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0254e3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.777806:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.777815:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009195ead4/0xd5401cbabc01f8b6 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5b4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0120 expref: 843 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.777818:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0254e3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.777845:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0254e3 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.777848:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009195ead4/0xd5401cbabc01f8b6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0120 expref: 843 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.777866:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc0254e3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.777876:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.777884:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.777889:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0254f8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.777892:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.777896:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.777900:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.777901:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0254f8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5b4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.777918:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000004:00080000:2.0:1783520935.777927:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c551180/2, rc 0, transno 4294975721 00010000:00010000:3.0:1783520935.777931:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0254f1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.777941:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c551180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993088768:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.777945:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0254f1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.777953:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435079f1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+843:2388:x1870153002382848:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 204us (431us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.777960:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0254f1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.777967:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35816 00000100:00100000:0.0:1783520935.777969:0:6496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435079f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt_rdpg00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+26:251795:x1870153002383104:12345-192.168.204.33@tcp:35:dd.0 Request processed in 200us (315us total) trans 8589957895 rc 0/0 00000100:00100000:0.0:1783520935.777983:0:6496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 9625 00010000:00010000:3.0:1783520935.777991:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520935.777999:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350794d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993090816:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.778005:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.778010:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0254f1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.778015:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.778020:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0254f1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.778029:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0254f1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.778041:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.778056:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0254f1 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.778064:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0254f1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.778085:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.778088:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.778092:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993090816 00010000:00010000:0.0:1783520935.778092:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.778098:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993090816 00000100:00100000:2.0:1783520935.778130:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44227 00000100:00100000:2.0:1783520935.778139:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350794df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+158:3626:x1870152993090816:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.778154:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1464 count 1 00010000:00010000:2.0:1783520935.778166:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025514 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5b8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.778175:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000068b21b6a/0xd5401cbabc01f8c4 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f013c expref: 842 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.778180:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b8:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.778182:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.778185:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025514 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.778193:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350794d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+159:3625:x1870152993089152:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3589us (3745us total) trans 4294975723 rc 0/0 00000100:00100000:0.0:1783520935.778200:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44220 00010000:00010000:2.0:1783520935.778207:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000068b21b6a/0xd5401cbabc01f8c4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f013c expref: 842 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.778216:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000068b21b6a/0xd5401cbabc01f8c4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f013c expref: 842 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.778224:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515344000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993090944:192.168.204.33@tcp:104: 00000004:00080000:1.0:1783520935.778230:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c551880/2, rc 0, transno 4294975723 00000100:00100000:1.0:1783520935.778242:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c551880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993089152:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.778244:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.778275:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517244e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+158:3628:x1870152993089024:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3631us (3741us total) trans 4294975722 rc 0/0 00000100:00100000:2.0:1783520935.778277:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514219780 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.778287:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94361103aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993091072:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.778288:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44222 00000100:00100000:2.0:1783520935.778300:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002383232 00000100:00100000:2.0:1783520935.778306:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002383232 00010000:00010000:2.0:1783520935.778328:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b23a7b29/0xd5401cbabc01fd55 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0127 expref: 843 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.778331:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c552300/2, rc 0, transno 4294975722 00000100:00100000:1.0:1783520935.778333:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.778341:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.778392:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002383360 00000100:00100000:2.0:1783520935.778396:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002383360 00000100:00100000:0.0:1783520935.778410:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.778413:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000084912c38/0xd5401cbabc01f8bd lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f012e expref: 844 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.778414:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c552300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993089024:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.778439:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35817 00000100:00100000:0.0:1783520935.778441:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002383488 00000100:00100000:0.0:1783520935.778446:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002383488 00000100:00100000:2.0:1783520935.778448:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+844:2388:x1870153002383232:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.778453:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35818 00010000:00010000:2.0:1783520935.778458:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.778463:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad0700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+845:2388:x1870153002383360:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.778466:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b23a7b29/0xd5401cbabc01fd55 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5b5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0127 expref: 845 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.778468:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c6f4370a/0xd5401cbabc01fd5c lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0135 expref: 845 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.778472:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.778480:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000084912c38/0xd5401cbabc01f8bd lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5b6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f012e expref: 845 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.778519:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000084912c38/0xd5401cbabc01f8bd lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f012e expref: 845 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.778526:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b23a7b29/0xd5401cbabc01fd55 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0127 expref: 845 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.778526:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35819 00000100:00100000:3.0:1783520935.778542:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993091200:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.778542:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357f850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+845:2388:x1870153002383488:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.778546:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b6:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.778550:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.778555:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b6:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.778557:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.778560:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.778562:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025506 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.778564:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0254ff lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.778569:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c6f4370a/0xd5401cbabc01fd5c lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5b7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0135 expref: 845 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.778584:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.778587:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025506 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5b6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.778593:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0254ff lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5b5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.778617:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.778622:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c6f4370a/0xd5401cbabc01fd5c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0135 expref: 845 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.778627:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.778629:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993091072 00000100:00100000:3.0:1783520935.778635:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993091072 00010000:00010000:0.0:1783520935.778638:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.778643:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.778648:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02550d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.778651:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad0700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+845:2388:x1870153002383360:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 187us (313us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.778659:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+845:2388:x1870153002383232:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 210us (416us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.778663:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993091200 00000100:00100000:3.0:1783520935.778669:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993091200 00000100:00100000:1.0:1783520935.778670:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35818 00010000:00010000:0.0:1783520935.778671:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02550d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5b7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.778681:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35817 00010000:00010000:0.0:1783520935.778695:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.778701:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44228 00000100:00100000:1.0:1783520935.778709:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44229 00000100:00100000:3.0:1783520935.778714:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cba1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+159:3627:x1870152993091072:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.778722:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357f850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+843:2388:x1870153002383488:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 182us (311us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.778723:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+159:3628:x1870152993091200:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520935.778731:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1465 count 1 00010000:00010000:2.0:1783520935.778736:0:8410:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b23a7b29/0xd5401cbabc01fd55 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0127 expref: 843 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.778739:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35819 00002000:00080000:1.0:1783520935.778743:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1466 count 1 00010000:00010000:3.0:1783520935.778749:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02551b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5b9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.778762:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350794e300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993090176:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.778763:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025522 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5ba:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.778768:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002ed79db9/0xd5401cbabc01fd63 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0143 expref: 841 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.778783:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0254ff lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.778791:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005fc8805e/0xd5401cbabc022af1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f31e1 expref: 841 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.778792:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b9:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.778802:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.778803:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0254ff lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.778810:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x5ba:0x0].0x0 since=36s v=(0 0) 00010000:00010000:3.0:1783520935.778813:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02551b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.778821:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ba:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.778832:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025522 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.778845:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0254ff lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.778856:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0254ff lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.778859:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002ed79db9/0xd5401cbabc01fd63 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0143 expref: 841 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.778865:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0254ff lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.778867:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005fc8805e/0xd5401cbabc022af1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31e1 expref: 841 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.778878:0:8408:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009195ead4/0xd5401cbabc01f8b6 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0120 expref: 841 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.778893:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002ed79db9/0xd5401cbabc01fd63 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0143 expref: 841 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.778894:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.778898:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005fc8805e/0xd5401cbabc022af1 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31e1 expref: 841 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.778905:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993089920:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.778924:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0254f8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.778928:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c7480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993091328:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.778930:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c550000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993091456:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.778945:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0254f8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.778958:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0254f8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.778967:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0254f8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.778974:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0254f8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.778987:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435129679c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.778987:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6300 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.778990:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:2.0:1783520935.779001:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943611039500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+160:3627:x1870152993090048:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3272us (3354us total) trans 4294975724 rc 0/0 00010000:00010000:1.0:1783520935.779007:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00000100:00100000:2.0:1783520935.779012:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44224 00010000:00010000:1.0:1783520935.779015:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0254ff lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.779024:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943611038700/2, rc 0, transno 4294975724 00010000:00010000:1.0:1783520935.779025:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.779033:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0254ff lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.779042:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943611038700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993090048:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.779043:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0254ea lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.779043:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0254ff lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.779067:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0254ea lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.779073:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943611038e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+160:3626:x1870152993089792:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3763us (3847us total) trans 4294975725 rc 0/0 00010000:00010000:1.0:1783520935.779074:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0254ff lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.779085:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0254ea lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.779094:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44223 00010000:00010000:3.0:1783520935.779100:0:20415:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c6f4370a/0xd5401cbabc01fd5c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0135 expref: 840 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.779114:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0254ff lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.779115:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0254ea lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.779116:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:3.0:1783520935.779123:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f3b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993090688:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.779125:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0254f8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.779146:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02550d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.779151:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfa400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.779155:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.779158:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0254f8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.779159:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.779165:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02550d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.779168:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.779169:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.779178:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0254ea lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.779178:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0254f8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.779179:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02550d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.779183:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.779199:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.779206:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0254ea lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520935.779207:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cba3b80/2, rc 0, transno 4294975725 00010000:00010000:3.0:1783520935.779208:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02550d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.779212:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0254f8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.779228:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993089792:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.779233:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0254ea lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.779240:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02550d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.779241:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc0254f8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.779242:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1783520935.779253:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993091584:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.779257:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0254ea lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.779272:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc0254ea lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.779279:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.779283:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.779288:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.779289:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.779292:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.779295:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.779296:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.779306:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.779309:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1783520935.779322:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993091712:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.779336:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.779342:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02550d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.779351:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.779355:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02550d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.779394:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.779402:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02550d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.779414:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02550d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.779422:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02550d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.779430:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993091584 00000100:00100000:1.0:1783520935.779436:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993091584 00010000:00010000:0.0:1783520935.779439:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.779444:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c553800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+162:3627:x1870152993088896:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4822us (4970us total) trans 4294975727 rc 0/0 00010000:00010000:0.0:1783520935.779444:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.779449:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.779455:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44221 00000100:00100000:1.0:1783520935.779468:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993091712 00000100:00100000:1.0:1783520935.779474:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993091712 00000100:00100000:0.0:1783520935.779475:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44230 00000100:00100000:0.0:1783520935.779488:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515345500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+162:3626:x1870152993091584:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.779513:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1467 count 1 00000100:00100000:1.0:1783520935.779514:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44231 00010000:00010000:2.0:1783520935.779517:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000084912c38/0xd5401cbabc01f8bd lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f012e expref: 839 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.779527:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638997800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+162:3625:x1870152993091712:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.779531:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350794c000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993090560:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.779531:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025529 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5bb:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.779541:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1468 count 1 00010000:00010000:0.0:1783520935.779550:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004305f2c3/0xd5401cbabc01fd71 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0151 expref: 838 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.779553:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025506 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.779553:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025530 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5bc:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.779560:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c550700/2, rc 0, transno 4294975727 00010000:00010000:0.0:1783520935.779568:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5bb:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.779578:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5bb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.779581:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025506 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.779585:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000062ef7f3/0xd5401cbabc01f8f5 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0158 expref: 838 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.779586:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025529 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.779594:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025506 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.779610:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5bc:0x0].0x0 since=37s v=(0 0) 00000100:00100000:3.0:1783520935.779613:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c550700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993088896:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.779620:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5bc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.779622:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025506 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.779628:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025530 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.779629:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004305f2c3/0xd5401cbabc01fd71 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0151 expref: 838 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.779641:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025506 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.779653:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:0.0:1783520935.779659:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004305f2c3/0xd5401cbabc01fd71 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0151 expref: 838 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.779664:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993091968:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.779671:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000062ef7f3/0xd5401cbabc01f8f5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0158 expref: 838 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.779672:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.779687:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520935.779687:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993091840:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.779688:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000062ef7f3/0xd5401cbabc01f8f5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0158 expref: 838 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.779694:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025506 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.779698:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.779719:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.779726:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025506 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.779733:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943638995500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993092096:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.779738:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.779741:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025506 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.779752:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993091968 00010000:00010000:2.0:1783520935.779756:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025506 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.779757:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993091968 00000100:00100000:1.0:1783520935.779767:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6f00 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.779768:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025506 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.779793:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.779797:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44232 00010000:00010000:2.0:1783520935.779797:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.779803:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.779828:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515344000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993090944:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.779843:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.779846:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025514 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.779851:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+163:3628:x1870152993091968:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.779866:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002383616 00002000:00080000:3.0:1783520935.779869:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1469 count 1 00000100:00100000:1.0:1783520935.779870:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002383616 00010000:00010000:3.0:1783520935.779882:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025537 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5bd:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.779890:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000068b21b6a/0xd5401cbabc01f8c4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f013c expref: 839 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.779895:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000039324521/0xd5401cbabc01fd6a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f015f expref: 839 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.779916:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35820 00000100:00100000:1.0:1783520935.779923:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638996a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+839:2388:x1870153002383616:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.779933:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.779938:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000068b21b6a/0xd5401cbabc01f8c4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f013c expref: 839 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.779944:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5bd:0x0].0x0 since=37s v=(0 0) 00000100:00100000:2.0:1783520935.779948:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94361103b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+164:3627:x1870152993090304:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3799us (3883us total) trans 4294975728 rc 0/0 00010000:00010000:3.0:1783520935.779954:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5bd:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.779959:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44225 00010000:00010000:1.0:1783520935.779959:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000068b21b6a/0xd5401cbabc01f8c4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5b8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f013c expref: 839 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.779963:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025537 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.779963:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350794ed80/2, rc 0, transno 4294975728 00000100:00100000:0.0:1783520935.779979:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350794ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993090304:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.779995:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000068b21b6a/0xd5401cbabc01f8c4 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5b8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f013c expref: 839 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.780000:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000039324521/0xd5401cbabc01fd6a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f015f expref: 839 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.780018:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000039324521/0xd5401cbabc01fd6a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f015f expref: 838 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.780019:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.780028:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b8:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.780032:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c5500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993092224:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.780034:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025514 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.780050:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025514 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.780058:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967e40 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.780075:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.780103:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638996a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+838:2388:x1870153002383616:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 178us (260us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.780105:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993092352:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.780105:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025514 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.780122:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35820 00010000:00010000:2.0:1783520935.780124:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025514 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.780136:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025514 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.780146:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.780146:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025514 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.780163:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.780174:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.780180:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025514 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.780190:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.780195:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025514 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.780204:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025514 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.780217:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025514 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.780227:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025514 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.780248:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.780251:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350794df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+164:3626:x1870152993090816:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2111us (2211us total) trans 4294975729 rc 0/0 00010000:00010000:0.0:1783520935.780252:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.780253:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993092352 00010000:00010000:0.0:1783520935.780257:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.780260:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993092352 00000100:00100000:2.0:1783520935.780265:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44227 00000100:00100000:2.0:1783520935.780288:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44233 00000100:00100000:2.0:1783520935.780299:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+164:3628:x1870152993092352:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.780313:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1470 count 1 00010000:00010000:2.0:1783520935.780325:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02553e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5be:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.780329:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943611038a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+164:3626:x1870152993090432:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3940us (4043us total) trans 4294975726 rc 0/0 00000100:00100000:3.0:1783520935.780346:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44226 00010000:00010000:2.0:1783520935.780349:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000084c21622/0xd5401cbabc01f8fc lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0166 expref: 837 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.780393:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5be:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.780398:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5be:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.780402:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02553e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.780416:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350794d500/2, rc 0, transno 4294975729 00010000:00010000:2.0:1783520935.780423:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000084c21622/0xd5401cbabc01f8fc lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0166 expref: 837 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.780432:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350794d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993090816:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.780437:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000084c21622/0xd5401cbabc01f8fc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0166 expref: 837 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.780449:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435079f1c00/2, rc 0, transno 4294975726 00000100:00100000:2.0:1783520935.780451:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515344000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993092480:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.780458:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993090432:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.780466:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1783520935.780476:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943638994700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993092608:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.780487:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1783520935.780511:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943638996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993092736:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.780521:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514219840 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.780522:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.780545:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993092608 00000100:00100000:2.0:1783520935.780551:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993092608 00000100:00100000:1.0:1783520935.780557:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.780585:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993092736 00000100:00100000:2.0:1783520935.780589:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993092736 00000100:00100000:0.0:1783520935.780590:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44234 00000100:00100000:0.0:1783520935.780601:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+165:3626:x1870152993092608:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.780611:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44235 00002000:00080000:0.0:1783520935.780615:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1471 count 1 00000100:00100000:2.0:1783520935.780623:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638996680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+165:3625:x1870152993092736:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.780629:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025545 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5bf:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.780639:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1472 count 1 00010000:00010000:0.0:1783520935.780644:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000090e78c63/0xd5401cbabc01fd78 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f016d expref: 837 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.780654:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02554c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5c0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.780654:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5bf:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.780661:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5bf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.780671:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025545 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.780681:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002b11a42b/0xd5401cbabc01f903 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0174 expref: 837 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.780689:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c0:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.780693:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.780698:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000090e78c63/0xd5401cbabc01fd78 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f016d expref: 837 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.780699:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02554c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.780725:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000090e78c63/0xd5401cbabc01fd78 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f016d expref: 837 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.780740:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002b11a42b/0xd5401cbabc01f903 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0174 expref: 837 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.780740:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad1180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993092864:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.780757:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002b11a42b/0xd5401cbabc01f903 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0174 expref: 837 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.780770:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515344380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993092992:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.780780:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124706c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.780800:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514219780 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.781069:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c7480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993091328:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.781086:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02551b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.781110:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c550000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993091456:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.781122:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993091840:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.781127:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943638995500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993092096:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.781128:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025522 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.781141:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025529 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.781150:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025530 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.781653:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.781671:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c5500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993092224:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.781675:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.781690:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025537 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.781702:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002383744 00000100:00100000:3.0:1783520935.781709:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.781709:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002383744 00000100:00100000:1.0:1783520935.781722:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002384000 00000100:00100000:1.0:1783520935.781727:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002384000 00010000:00010000:2.0:1783520935.781733:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002ed79db9/0xd5401cbabc01fd63 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0143 expref: 839 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.781734:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002383872 00000100:00100000:0.0:1783520935.781739:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002383872 00010000:00010000:1.0:1783520935.781744:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000062ef7f3/0xd5401cbabc01f8f5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0158 expref: 839 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.781757:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005fc8805e/0xd5401cbabc022af1 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31e1 expref: 840 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.781783:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35821 00000100:00100000:1.0:1783520935.781790:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35822 00000100:00100000:0.0:1783520935.781796:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35823 00000100:00100000:1.0:1783520935.781802:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+840:2388:x1870153002384000:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.781808:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c7480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+840:2388:x1870153002383744:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.781810:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c6300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+840:2388:x1870153002383872:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.781815:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.781823:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.781824:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000062ef7f3/0xd5401cbabc01f8f5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5bc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0158 expref: 840 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.781824:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.781831:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002ed79db9/0xd5401cbabc01fd63 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5b9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0143 expref: 840 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.781835:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005fc8805e/0xd5401cbabc022af1 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5ba:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31e1 expref: 840 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.781845:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.781851:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000062ef7f3/0xd5401cbabc01f8f5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5bc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0158 expref: 840 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.781864:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002ed79db9/0xd5401cbabc01fd63 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5b9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0143 expref: 840 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.781865:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005fc8805e/0xd5401cbabc022af1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5ba:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31e1 expref: 840 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.781877:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000062ef7f3/0xd5401cbabc01f8f5 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5bc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0158 expref: 840 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.781878:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.781897:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5bc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.781898:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005fc8805e/0xd5401cbabc022af1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5ba:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31e1 expref: 839 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.781902:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002ed79db9/0xd5401cbabc01fd63 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5b9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0143 expref: 839 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.781902:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5bc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.781911:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025530 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.781918:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ba:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.781927:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ba:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.781929:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025530 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.781935:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.781935:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025522 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.781937:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.781948:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5b9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.781954:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02551b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.781956:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025522 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.781961:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.781976:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02551b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.781987:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.781993:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+837:2388:x1870153002384000:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 195us (317us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.781998:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025522 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.782003:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.782006:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35822 00000100:00100000:0.0:1783520935.782012:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c6300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+837:2388:x1870153002383872:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 204us (303us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.782019:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35823 00000100:00100000:2.0:1783520935.782030:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c7480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+835:2388:x1870153002383744:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 237us (375us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.782030:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002384128 00000100:00100000:0.0:1783520935.782034:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002384256 00000100:00100000:1.0:1783520935.782036:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002384128 00000100:00100000:0.0:1783520935.782039:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002384256 00000100:00100000:2.0:1783520935.782043:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35821 00010000:00010000:0.0:1783520935.782061:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000039324521/0xd5401cbabc01fd6a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f015f expref: 836 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.782062:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004305f2c3/0xd5401cbabc01fd71 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0151 expref: 836 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.782064:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002384384 00010000:00010000:3.0:1783520935.782069:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025522 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.782070:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002384384 00010000:00010000:2.0:1783520935.782087:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000084c21622/0xd5401cbabc01f8fc lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0166 expref: 837 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.782090:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025522 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.782092:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35824 00000100:00100000:0.0:1783520935.782105:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350d165f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+837:2387:x1870153002384256:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.782111:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025522 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.782112:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35825 00010000:00010000:0.0:1783520935.782116:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.782123:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000039324521/0xd5401cbabc01fd6a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5bd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f015f expref: 837 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.782129:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35826 00000100:00100000:1.0:1783520935.782129:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c5500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+837:2388:x1870153002384128:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.782135:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.782141:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000039324521/0xd5401cbabc01fd6a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5bd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f015f expref: 837 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.782143:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520935.782145:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357dc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+837:2388:x1870153002384384:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.782151:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004305f2c3/0xd5401cbabc01fd71 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5bb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0151 expref: 837 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.782158:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.782166:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000084c21622/0xd5401cbabc01f8fc lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5be:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0166 expref: 837 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.782168:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000039324521/0xd5401cbabc01fd6a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5bd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f015f expref: 837 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.782170:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004305f2c3/0xd5401cbabc01fd71 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5bb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0151 expref: 837 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.782196:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000084c21622/0xd5401cbabc01f8fc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5be:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0166 expref: 837 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.782199:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5bd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.782201:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004305f2c3/0xd5401cbabc01fd71 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5bb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0151 expref: 836 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.782208:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5bd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.782217:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025537 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.782226:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025537 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.782228:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5be:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.782229:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5bb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.782236:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5bb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.782239:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5be:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.782243:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025529 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.782243:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.782246:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02553e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5be:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.782266:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025529 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.782269:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02553e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5be:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.782271:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350d165f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+835:2387:x1870153002384256:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 167us (394us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.782291:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35824 00000100:00100000:3.0:1783520935.782296:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+166:3628:x1870152993091200:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3575us (3713us total) trans 4294975730 rc 0/0 00010000:00010000:2.0:1783520935.782297:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.782300:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.782308:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44229 00000100:00100000:2.0:1783520935.782322:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357dc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+834:2388:x1870153002384384:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 180us (389us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.782323:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025537 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.782328:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c5500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+834:2388:x1870153002384128:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 204us (486us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.782341:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35826 00010000:00010000:0.0:1783520935.782343:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025537 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.782350:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435079f2d80/2, rc 0, transno 4294975730 00000100:00100000:1.0:1783520935.782350:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35825 00010000:00010000:0.0:1783520935.782353:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025537 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.782390:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993091200:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.782410:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025537 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.782423:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520935.782424:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02551b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.782431:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1783520935.782439:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025522 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.782440:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02551b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.782465:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02551b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.782468:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7400 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:3.0:1783520935.782474:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350d167800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993093120:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.782477:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025522 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.782479:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02551b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.782494:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+166:3628:x1870152993091968:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2687us (2800us total) trans 4294975731 rc 0/0 00010000:00010000:1.0:1783520935.782512:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025522 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.782518:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:3.0:1783520935.782520:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.782523:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44232 00010000:00010000:1.0:1783520935.782538:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025522 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.782541:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993093120 00000100:00100000:0.0:1783520935.782546:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993093120 00010000:00010000:1.0:1783520935.782552:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc025522 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.782575:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00000100:00100000:0.0:1783520935.782576:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44236 00010000:00010000:1.0:1783520935.782579:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025537 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.782582:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e1c6680/2, rc 0, transno 4294975731 00000100:00100000:0.0:1783520935.782589:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350d165880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+167:3627:x1870152993093120:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.782591:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.782595:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025537 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.782598:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993091968:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.782608:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025537 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.782609:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1473 count 1 00000100:00100000:2.0:1783520935.782611:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cba1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+167:3627:x1870152993091072:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3901us (4280us total) trans 4294975732 rc 0/0 00010000:00010000:0.0:1783520935.782627:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025553 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5c1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.782629:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44228 00010000:00010000:1.0:1783520935.782631:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025537 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.782650:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025537 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.782651:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350d165180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993093248:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.782651:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.782652:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002228b00e/0xd5401cbabc01fd7f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f017b expref: 832 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.782660:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02551b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.782676:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c1:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.782678:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4600 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:3.0:1783520935.782683:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.782684:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02551b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.782684:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.782691:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025553 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.782694:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.782696:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02551b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.782699:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.782703:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.782719:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02551b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.782719:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002228b00e/0xd5401cbabc01fd7f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f017b expref: 832 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.782722:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993093248 00000100:00100000:3.0:1783520935.782728:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993093248 00010000:00010000:2.0:1783520935.782739:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02551b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.782742:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002228b00e/0xd5401cbabc01fd7f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f017b expref: 832 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.782756:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.782762:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.782763:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993093376:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.782766:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44237 00010000:00010000:2.0:1783520935.782766:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.782769:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.782782:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350d166680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+167:3628:x1870152993093248:12345-0@lo:6:osp-syn-1-0.0 00000004:00080000:2.0:1783520935.782795:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94361103aa00/2, rc 0, transno 4294975732 00002000:00080000:3.0:1783520935.782800:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1474 count 1 00000100:00100000:2.0:1783520935.782811:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94361103aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993091072:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.782814:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470180 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.782816:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc02555a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5c2:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.782830:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009342c621/0xd5401cbabc01f97a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0262 expref: 832 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.782832:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.782840:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c2:0x0].0x0 since=37s v=(0 0) 00000100:00100000:0.0:1783520935.782844:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002384512 00010000:00010000:3.0:1783520935.782849:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c2:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.782850:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.782850:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002384512 00010000:00010000:3.0:1783520935.782859:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc02555a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.782868:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000090e78c63/0xd5401cbabc01fd78 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f016d expref: 833 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.782869:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002384640 00000100:00100000:2.0:1783520935.782875:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002384640 00010000:00010000:2.0:1783520935.782892:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002b11a42b/0xd5401cbabc01f903 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0174 expref: 834 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.782899:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35827 00000100:00100000:0.0:1783520935.782908:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638996a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+834:2388:x1870153002384512:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.782915:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.782920:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000090e78c63/0xd5401cbabc01fd78 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5bf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f016d expref: 834 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.782924:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35828 00000100:00100000:2.0:1783520935.782937:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638995500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+834:2388:x1870153002384640:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.782937:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000090e78c63/0xd5401cbabc01fd78 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5bf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f016d expref: 834 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.782944:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009342c621/0xd5401cbabc01f97a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0262 expref: 834 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.782948:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.782956:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5bf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.782957:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002b11a42b/0xd5401cbabc01f903 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5c0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0174 expref: 834 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.782967:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5bf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.782970:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009342c621/0xd5401cbabc01f97a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0262 expref: 834 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.782975:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025545 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5bf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.782976:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002b11a42b/0xd5401cbabc01f903 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5c0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0174 expref: 834 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.782977:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025530 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.782996:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350d167100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993093632:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.783006:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025530 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.783008:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025545 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5bf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.783027:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.783031:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025530 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.783032:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.783037:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.783043:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02554c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.783047:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025530 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.783062:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02554c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5c0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.783072:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520935.783075:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435129670c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.783076:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638996a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+834:2388:x1870153002384512:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 168us (325us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.783079:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.783086:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35827 00010000:00010000:0.0:1783520935.783103:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:2.0:1783520935.783109:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638995500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+833:2388:x1870153002384640:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 174us (264us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.783109:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025530 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.783124:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6800 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520935.783126:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35828 00010000:00010000:0.0:1783520935.783128:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025530 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.783136:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025530 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.783145:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025530 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.783153:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc025530 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.783165:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515345180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993093504:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.783172:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.783173:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638997800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+168:3625:x1870152993091712:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3647us (3812us total) trans 4294975733 rc 0/0 00010000:00010000:0.0:1783520935.783177:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.783182:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.783185:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44231 00000100:00100000:2.0:1783520935.783199:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.783219:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993093504 00000100:00100000:0.0:1783520935.783224:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993093504 00000004:00080000:2.0:1783520935.783239:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e1c4a80/2, rc 0, transno 4294975733 00000100:00100000:0.0:1783520935.783252:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44238 00000100:00100000:2.0:1783520935.783253:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993091712:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.783265:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515346d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+168:3625:x1870152993093504:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.783279:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1475 count 1 00010000:00010000:0.0:1783520935.783292:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025561 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5c3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.783305:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000018a5ed86/0xd5401cbabc01f981 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0270 expref: 832 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.783312:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c3:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.783315:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c3:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.783318:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515347480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993093760:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.783321:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025561 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.783335:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000084c21622/0xd5401cbabc01f8fc lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0166 expref: 832 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.783348:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000018a5ed86/0xd5401cbabc01f981 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0270 expref: 831 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.783349:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515344000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993092480:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.783377:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.783394:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000018a5ed86/0xd5401cbabc01f981 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0270 expref: 831 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.783398:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02553e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.783411:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993093888:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.783413:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02553e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.783423:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02553e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.783432:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02553e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.783441:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02553e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.783459:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993093760 00010000:00010000:2.0:1783520935.783460:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.783465:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470f00 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.783466:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993093760 00010000:00010000:0.0:1783520935.783491:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025529 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.783513:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025529 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.783517:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44239 00010000:00010000:0.0:1783520935.783529:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025529 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.783531:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635603850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+170:3626:x1870152993093760:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.783540:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.783547:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025529 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.783549:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02553e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520935.783551:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1476 count 1 00000100:00100000:2.0:1783520935.783565:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+170:3628:x1870152993092352:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3267us (3418us total) trans 4294975734 rc 0/0 00010000:00010000:3.0:1783520935.783566:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025568 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5c4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.783573:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185c00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520935.783578:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44233 00010000:00010000:0.0:1783520935.783579:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.783580:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02553e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.783592:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02553e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.783602:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.783605:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02553e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.783609:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025529 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.783621:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02553e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.783633:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfa200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.783642:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025529 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.783649:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002a69a8f5/0xd5401cbabc01f988 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f027e expref: 831 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.783656:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025529 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.783656:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.783662:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c4:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.783662:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.783664:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515345500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+170:3626:x1870152993091584:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4178us (4378us total) trans 4294975735 rc 0/0 00010000:00010000:1.0:1783520935.783666:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.783672:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.783682:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025568 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.783682:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44230 00010000:00010000:2.0:1783520935.783689:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025529 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.783693:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e1c4e00/2, rc 0, transno 4294975735 00010000:00010000:3.0:1783520935.783706:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002a69a8f5/0xd5401cbabc01f988 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f027e expref: 831 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.783706:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025529 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.783710:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993091584:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.783723:0:8408:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000090e78c63/0xd5401cbabc01fd78 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f016d expref: 831 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.783735:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002a69a8f5/0xd5401cbabc01f988 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f027e expref: 831 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.783744:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.783749:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350d166d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993094016:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.783749:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.783753:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.783753:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad1180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993092864:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.783764:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943638994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993094144:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.783772:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025545 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520935.783779:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e1c5f80/2, rc 0, transno 4294975734 00010000:00010000:0.0:1783520935.783783:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025545 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.783794:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.783795:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993092352:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.783795:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025545 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.783811:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025545 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.783819:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025545 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.783835:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967840 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.783837:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.783839:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993094144 00000100:00100000:1.0:1783520935.783844:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993094144 00010000:00010000:0.0:1783520935.783849:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.783856:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025545 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.783862:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515344000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993094272:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.783866:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.783872:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025545 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.783873:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44240 00000100:00100000:1.0:1783520935.783882:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638995180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+170:3627:x1870152993094144:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.783885:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025545 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.783894:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1477 count 1 00010000:00010000:0.0:1783520935.783900:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025545 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.783902:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.783905:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02556f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5c5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.783924:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc025545 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.783927:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007a542845/0xd5401cbabc01f98f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f028c expref: 830 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.783944:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993094272 00010000:00010000:1.0:1783520935.783946:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c5:0x0].0x0 since=37s v=(0 0) 00000100:00100000:2.0:1783520935.783949:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993094272 00010000:00010000:1.0:1783520935.783952:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.783953:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.783958:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.783959:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02556f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.783961:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.783976:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44241 00000100:00100000:2.0:1783520935.783988:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c550000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+171:3628:x1870152993094272:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.783988:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007a542845/0xd5401cbabc01f98f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f028c expref: 830 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.783990:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993093376:192.168.204.33@tcp:104: 00002000:00080000:2.0:1783520935.784006:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1478 count 1 00010000:00010000:1.0:1783520935.784008:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007a542845/0xd5401cbabc01f98f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f028c expref: 830 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.784010:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025553 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.784017:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025576 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5c6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.784036:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993094400:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.784042:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000aa2234eb/0xd5401cbabc022b29 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3219 expref: 830 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.784055:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x5c6:0x0].0x0 since=36s v=(0 0) 00010000:00010000:2.0:1783520935.784059:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.784063:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025576 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.784070:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6840 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.784085:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.784097:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000aa2234eb/0xd5401cbabc022b29 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3219 expref: 830 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.784109:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000aa2234eb/0xd5401cbabc022b29 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3219 expref: 830 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.784122:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350794c000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993094528:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.784128:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350d167100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993093632:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.784145:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002384768 00010000:00010000:3.0:1783520935.784148:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc02555a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.784149:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002384768 00000100:00100000:0.0:1783520935.784155:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+172:3626:x1870152993092608:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3555us (3632us total) trans 4294975736 rc 0/0 00000100:00100000:2.0:1783520935.784161:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895840 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.784163:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44234 00010000:00010000:1.0:1783520935.784170:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009342c621/0xd5401cbabc01f97a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5c2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0262 expref: 831 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.784199:0:8410:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002b11a42b/0xd5401cbabc01f903 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0174 expref: 831 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.784209:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35829 00000100:00100000:2.0:1783520935.784220:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515344380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993092992:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.784223:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350d166a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+830:2388:x1870153002384768:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.784234:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.784239:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02554c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.784241:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009342c621/0xd5401cbabc01f97a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5c2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0262 expref: 830 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.784265:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993093888:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.784268:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02554c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.784270:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009342c621/0xd5401cbabc01f97a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5c2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0262 expref: 830 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.784290:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025561 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.784291:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02554c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.784301:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009342c621/0xd5401cbabc01f97a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5c2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0262 expref: 830 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.784317:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02554c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.784323:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.784332:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.784341:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc02555a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.784342:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02554c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.784387:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc02555a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.784394:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.784403:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.784409:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.784410:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02554c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.784420:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.784424:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02554c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.784433:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02554c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.784447:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350d166a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+829:2388:x1870153002384768:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 228us (364us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.784449:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02554c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.784456:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.784457:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35829 00010000:00010000:0.0:1783520935.784467:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02554c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.784475:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002384896 00000100:00100000:1.0:1783520935.784479:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002384896 00000100:00100000:2.0:1783520935.784489:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638996680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+172:3625:x1870152993092736:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3868us (3933us total) trans 4294975737 rc 0/0 00010000:00010000:0.0:1783520935.784495:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.784507:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002228b00e/0xd5401cbabc01fd7f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f017b expref: 829 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.784512:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.784516:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44235 00010000:00010000:0.0:1783520935.784518:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.784541:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35830 00000100:00100000:3.0:1783520935.784549:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.784556:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350d167100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+829:2388:x1870153002384896:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.784562:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002385024 00010000:00010000:1.0:1783520935.784565:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520935.784568:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002385024 00010000:00010000:1.0:1783520935.784573:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002228b00e/0xd5401cbabc01fd7f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5c1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f017b expref: 829 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.784591:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000018a5ed86/0xd5401cbabc01f981 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0270 expref: 830 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.784591:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002228b00e/0xd5401cbabc01fd7f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5c1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f017b expref: 830 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.784622:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002228b00e/0xd5401cbabc01fd7f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5c1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f017b expref: 830 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.784631:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35831 00010000:00010000:1.0:1783520935.784637:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c1:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.784642:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350d165f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+829:2390:x1870153002385024:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.784644:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.784652:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.784652:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025553 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.784662:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000018a5ed86/0xd5401cbabc01f981 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5c3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0270 expref: 829 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.784663:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025553 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.784670:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943638996d80/2, rc 0, transno 4294975737 00000100:00100000:3.0:1783520935.784684:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943638996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993092736:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.784689:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000018a5ed86/0xd5401cbabc01f981 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5c3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0270 expref: 829 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.784698:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.784708:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000018a5ed86/0xd5401cbabc01f981 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5c3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0270 expref: 829 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.784725:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350d167100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+828:2388:x1870153002384896:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 173us (407us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.784729:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c3:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.784733:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35830 00010000:00010000:2.0:1783520935.784736:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c3:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.784740:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502250700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993094656:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.784743:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025561 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.784754:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025561 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.784771:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943638994700/2, rc 0, transno 4294975736 00010000:00010000:2.0:1783520935.784779:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.784784:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.784786:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350d166d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993094016:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.784791:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943638994700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993092608:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.784809:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025568 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.784811:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350d165f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+827:2390:x1870153002385024:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 170us (264us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.784816:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025553 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.784823:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35831 00000100:00100000:0.0:1783520935.784827:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993094656 00010000:00010000:1.0:1783520935.784834:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025553 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.784834:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993094656 00010000:00010000:1.0:1783520935.784845:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025553 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.784848:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025561 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.784855:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025553 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.784861:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025561 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.784865:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44242 00000100:00100000:0.0:1783520935.784878:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+172:3625:x1870152993094656:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.784885:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025561 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.784890:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.784891:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025561 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.784896:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1479 count 1 00010000:00010000:2.0:1783520935.784911:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:0.0:1783520935.784913:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02557d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5c7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.784927:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004304ce8a/0xd5401cbabc01f99d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f02a8 expref: 826 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.784936:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c7:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.784939:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.784944:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02557d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.784968:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004304ce8a/0xd5401cbabc01f99d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02a8 expref: 826 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.784980:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004304ce8a/0xd5401cbabc01f99d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02a8 expref: 826 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.784983:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350d165880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+174:3627:x1870152993093120:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2395us (2465us total) trans 4294975738 rc 0/0 00000100:00100000:3.0:1783520935.784984:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502253480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993094784:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.784991:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515346d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+174:3625:x1870152993093504:12345-0@lo:6:osp-syn-1-0.0 Request processed in 1727us (1791us total) trans 4294975739 rc 0/0 00000100:00100000:1.0:1783520935.785000:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44236 00000100:00100000:2.0:1783520935.785008:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44238 00000100:00100000:0.0:1783520935.785012:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993094912:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.785026:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.785031:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00000004:00080000:1.0:1783520935.785034:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350d167800/2, rc 0, transno 4294975738 00010000:00010000:2.0:1783520935.785036:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025553 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.785042:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfbc00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.785046:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025553 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.785046:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350d167800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993093120:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.785057:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025553 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.785063:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470f00 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.785067:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025553 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.785073:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025553 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.785086:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993094784 00000100:00100000:3.0:1783520935.785091:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993094784 00010000:00010000:2.0:1783520935.785092:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.785097:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025561 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.785101:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfa000 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520935.785104:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993095040:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.785106:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025561 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.785112:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.785122:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025561 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.785127:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44243 00010000:00010000:2.0:1783520935.785133:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025561 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.785140:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502250380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+173:3626:x1870152993094784:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.785143:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025561 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.785146:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00080000:3.0:1783520935.785161:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1480 count 1 00010000:00010000:2.0:1783520935.785162:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.785165:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.785169:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.785177:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025584 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5c8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.785179:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993095040 00000100:00100000:2.0:1783520935.785184:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993095040 00010000:00010000:1.0:1783520935.785186:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.785198:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000038bf5646/0xd5401cbabc01f9ab lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f02b6 expref: 826 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.785206:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c8:0x0].0x0 since=37s v=(0 0) 00000100:00100000:2.0:1783520935.785211:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44244 00010000:00010000:1.0:1783520935.785212:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc02555a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.785213:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c8:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.785224:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+174:3627:x1870152993095040:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.785225:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025584 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.785235:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc02555a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.785244:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1481 count 1 00010000:00010000:1.0:1783520935.785254:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc02555a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.785261:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02558b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5c9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.785262:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000038bf5646/0xd5401cbabc01f9ab lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02b6 expref: 826 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.785265:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc02555a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.785286:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000729114ae/0xd5401cbabc01f9a4 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f02c4 expref: 826 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.785293:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.785294:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c9:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.785300:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.785303:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.785306:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02558b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.785312:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc02555a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.785325:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000038bf5646/0xd5401cbabc01f9ab lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02b6 expref: 826 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.785334:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.785342:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc02555a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.785347:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502250000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993095168:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.785347:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000729114ae/0xd5401cbabc01f9a4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02c4 expref: 826 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.785414:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000729114ae/0xd5401cbabc01f9a4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02c4 expref: 826 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.785415:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc02555a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.785444:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350794e300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993095296:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.785444:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc02555a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.785458:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc02555a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.785468:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350d166680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+175:3628:x1870152993093248:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2689us (2787us total) trans 4294975740 rc 0/0 00000100:00100000:1.0:1783520935.785478:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44237 00010000:00010000:0.0:1783520935.785481:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.785485:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967840 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.785486:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895600 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.785486:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.785492:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520935.785522:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350d165180/2, rc 0, transno 4294975740 00000100:00100000:1.0:1783520935.785531:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350d165180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993093248:0@lo:6:osp-syn-1-0.0 00000004:00080000:3.0:1783520935.785538:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515345180/2, rc 0, transno 4294975739 00000100:00100000:2.0:1783520935.785544:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350794c000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993094528:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.785547:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515345180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993093504:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.785571:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025576 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.785590:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993095552:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.785595:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1783520935.785598:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002385152 00000100:00100000:2.0:1783520935.785603:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002385152 00000100:00100000:1.0:1783520935.785608:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993095424:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.785625:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002a69a8f5/0xd5401cbabc01f988 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f027e expref: 827 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.785640:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.785657:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35832 00000100:00100000:3.0:1783520935.785668:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.785672:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+827:2388:x1870153002385152:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.785683:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.785688:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002a69a8f5/0xd5401cbabc01f988 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5c4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f027e expref: 827 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.785694:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993095424 00000100:00100000:0.0:1783520935.785700:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993095424 00010000:00010000:2.0:1783520935.785713:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002a69a8f5/0xd5401cbabc01f988 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5c4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f027e expref: 827 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.785724:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.785729:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002a69a8f5/0xd5401cbabc01f988 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5c4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f027e expref: 827 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.785732:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993095552 00000100:00100000:0.0:1783520935.785737:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993095552 00010000:00010000:2.0:1783520935.785745:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.785750:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.785755:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025568 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.785755:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.785765:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44245 00010000:00010000:2.0:1783520935.785768:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025568 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.785778:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44246 00000100:00100000:0.0:1783520935.785778:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+176:3627:x1870152993095424:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.785790:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502253800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+176:3625:x1870152993095552:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.785796:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1482 count 1 00010000:00010000:2.0:1783520935.785798:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.785805:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002385280 00002000:00080000:1.0:1783520935.785807:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1483 count 1 00010000:00010000:0.0:1783520935.785810:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025592 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5ca:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.785812:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002385280 00010000:00010000:1.0:1783520935.785823:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025599 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5cb:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.785825:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004a29fb1b/0xd5401cbabc01f9b2 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f02d2 expref: 827 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.785832:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+827:2388:x1870153002385152:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 163us (722us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.785835:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000aa2234eb/0xd5401cbabc022b29 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3219 expref: 827 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.785854:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35832 00010000:00010000:0.0:1783520935.785854:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ca:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.785855:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000077791a40/0xd5401cbabc01f9b9 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f02e0 expref: 827 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.785864:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ca:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.785874:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5cb:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.785874:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025592 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.785875:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002385408 00000100:00100000:2.0:1783520935.785881:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002385408 00000100:00100000:3.0:1783520935.785882:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35833 00010000:00010000:1.0:1783520935.785883:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5cb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.785893:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025599 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.785898:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357e450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+827:2388:x1870153002385280:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.785901:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007a542845/0xd5401cbabc01f98f lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f028c expref: 827 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.785909:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004a29fb1b/0xd5401cbabc01f9b2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02d2 expref: 827 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.785915:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.785925:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000aa2234eb/0xd5401cbabc022b29 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5c6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3219 expref: 827 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.785930:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000077791a40/0xd5401cbabc01f9b9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02e0 expref: 827 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.785934:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004a29fb1b/0xd5401cbabc01f9b2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02d2 expref: 827 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.785943:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35834 00010000:00010000:1.0:1783520935.785944:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000077791a40/0xd5401cbabc01f9b9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02e0 expref: 827 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.785959:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9fb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+827:2390:x1870153002385408:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.785965:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993095680:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.785966:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9c000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993095808:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.785972:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.785980:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007a542845/0xd5401cbabc01f98f lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5c5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f028c expref: 827 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.785998:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007a542845/0xd5401cbabc01f98f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5c5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f028c expref: 827 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.785998:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6cc0 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.786002:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000aa2234eb/0xd5401cbabc022b29 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5c6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3219 expref: 827 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.786009:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470180 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.786022:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993094400:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.786032:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007a542845/0xd5401cbabc01f98f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5c5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f028c expref: 827 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.786036:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000aa2234eb/0xd5401cbabc022b29 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5c6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3219 expref: 827 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.786040:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02556f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5c5:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.786054:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.786063:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.786067:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.786068:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02556f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.786073:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520935.786082:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025576 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.786085:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02556f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.786105:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025576 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.786117:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.786122:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.786126:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02556f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.786141:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02556f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.786148:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9fb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+825:2390:x1870153002385408:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 191us (390us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.786153:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02556f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.786160:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35834 00010000:00010000:1.0:1783520935.786170:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02556f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.786179:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002385536 00000100:00100000:2.0:1783520935.786185:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002385536 00010000:00010000:1.0:1783520935.786200:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.786202:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004304ce8a/0xd5401cbabc01f99d lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02a8 expref: 825 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.786207:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.786217:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02556f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.786227:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.786231:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02556f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.786239:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35835 00010000:00010000:0.0:1783520935.786251:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02556f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.786253:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+825:2388:x1870153002385536:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.786261:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.786268:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.786273:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02556f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.786275:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004304ce8a/0xd5401cbabc01f99d lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5c7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02a8 expref: 825 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.786289:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638995180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+177:3627:x1870152993094144:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2406us (2496us total) trans 4294975741 rc 0/0 00000100:00100000:3.0:1783520935.786292:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357e450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+825:2388:x1870153002385280:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 395us (566us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.786298:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02556f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.786304:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004304ce8a/0xd5401cbabc01f99d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5c7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02a8 expref: 825 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.786306:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44240 00010000:00010000:2.0:1783520935.786331:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c7:0x0].0x0 since=0s v=(0 0) 00000004:00080000:1.0:1783520935.786332:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943638994a80/2, rc 0, transno 4294975741 00010000:00010000:0.0:1783520935.786334:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.786336:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.786338:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.786342:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943638994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993094144:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.786343:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02557d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.786344:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.786355:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35833 00010000:00010000:2.0:1783520935.786384:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02557d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5c7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.786399:0:9151:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004304ce8a/0xd5401cbabc01f99d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5c7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02a8 expref: 824 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.786406:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.786414:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993094912:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.786431:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993095936:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.786431:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02557d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.786436:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+823:2388:x1870153002385536:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 183us (317us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.786452:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35835 00010000:00010000:0.0:1783520935.786453:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02557d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.786464:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02557d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.786472:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.786477:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025576 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.786479:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02557d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.786517:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02557d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.786521:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025576 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.786536:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025576 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.786538:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.786548:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025576 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.786565:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:1.0:1783520935.786593:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993095936 00000100:00100000:1.0:1783520935.786599:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993095936 00000100:00100000:0.0:1783520935.786624:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+179:3625:x1870152993094656:12345-0@lo:6:osp-syn-1-0.0 Request processed in 1746us (1840us total) trans 4294975742 rc 0/0 00000100:00100000:1.0:1783520935.786634:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44247 00000100:00100000:0.0:1783520935.786641:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44242 00000100:00100000:1.0:1783520935.786645:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502252a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+179:3626:x1870152993095936:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.786661:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1484 count 1 00000100:00100000:2.0:1783520935.786670:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c550000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+178:3628:x1870152993094272:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2686us (2769us total) trans 4294975743 rc 0/0 00010000:00010000:1.0:1783520935.786674:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0255a0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5cc:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.786681:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502250700/2, rc 0, transno 4294975742 00000100:00100000:2.0:1783520935.786683:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44241 00010000:00010000:1.0:1783520935.786696:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b59af2c4/0xd5401cbabc01f9c0 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f02ee expref: 822 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.786700:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502250700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993094656:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.786713:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00000100:00100000:0.0:1783520935.786719:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.786720:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5cc:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.786722:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02557d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.786726:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5cc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.786737:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0255a0 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.786738:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.786744:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02557d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.786748:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502253b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993096064:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.786754:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.786764:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02557d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.786768:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b59af2c4/0xd5401cbabc01f9c0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02ee expref: 822 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.786781:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.786782:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02557d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.786782:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002385664 00010000:00010000:1.0:1783520935.786786:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b59af2c4/0xd5401cbabc01f9c0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02ee expref: 822 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.786788:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002385664 00010000:00010000:2.0:1783520935.786802:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02557d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.786808:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000038bf5646/0xd5401cbabc01f9ab lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02b6 expref: 823 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.786810:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025568 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.786812:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9c380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993096192:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.786822:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.786829:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025576 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.786834:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025568 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.786843:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184a00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520935.786845:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35836 00010000:00010000:2.0:1783520935.786850:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025576 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.786854:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025568 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.786861:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad1f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+823:2388:x1870153002385664:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.786862:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6300 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.786868:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025576 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.786875:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025568 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.786875:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.786877:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.786888:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000038bf5646/0xd5401cbabc01f9ab lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5c8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02b6 expref: 823 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.786890:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025576 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.786894:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.786913:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025576 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.786921:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000038bf5646/0xd5401cbabc01f9ab lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5c8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02b6 expref: 823 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.786923:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993096064 00010000:00010000:2.0:1783520935.786928:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520935.786931:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993096064 00010000:00010000:2.0:1783520935.786934:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025568 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.786940:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.786942:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.786945:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.786948:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025568 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.786951:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025584 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.786959:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025568 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.786960:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44248 00010000:00010000:0.0:1783520935.786970:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025584 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5c8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.786974:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502253100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+179:3627:x1870152993096064:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.786977:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025568 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.786990:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025568 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.786993:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1485 count 1 00010000:00010000:0.0:1783520935.786994:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.787010:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.787012:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0255a7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5cd:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.787013:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.787018:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.787023:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad1f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+823:2388:x1870153002385664:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 168us (309us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.787029:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000020772469/0xd5401cbabc01f9c7 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f02fc expref: 823 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.787035:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35836 00000100:00100000:3.0:1783520935.787039:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635603850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+179:3626:x1870152993093760:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3510us (3683us total) trans 4294975744 rc 0/0 00000004:00080000:2.0:1783520935.787045:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515344000/2, rc 0, transno 4294975743 00000100:00100000:3.0:1783520935.787053:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44239 00000100:00100000:0.0:1783520935.787053:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002385792 00000100:00100000:0.0:1783520935.787058:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002385792 00000100:00100000:2.0:1783520935.787062:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515344000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993094272:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.787065:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5cd:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.787070:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5cd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.787074:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0255a7 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.787076:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000729114ae/0xd5401cbabc01f9a4 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02c4 expref: 823 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.787106:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:1.0:1783520935.787110:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000020772469/0xd5401cbabc01f9c7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02fc expref: 823 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.787115:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35837 00000100:00100000:2.0:1783520935.787117:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350794c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993096320:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.787128:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+823:2389:x1870153002385792:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.787130:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000020772469/0xd5401cbabc01f9c7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02fc expref: 823 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.787141:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.787147:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9d500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993096448:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.787149:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000729114ae/0xd5401cbabc01f9a4 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5c9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02c4 expref: 823 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.787156:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.787169:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000729114ae/0xd5401cbabc01f9a4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5c9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02c4 expref: 823 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.787183:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.787187:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5c9:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.787188:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6840 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.787193:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02558b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.787203:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02558b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5c9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.787213:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993096320 00010000:00010000:0.0:1783520935.787217:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.787218:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993096320 00010000:00010000:2.0:1783520935.787229:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000729114ae/0xd5401cbabc01f9a4 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02c4 expref: 823 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.787243:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+822:2389:x1870153002385792:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 116us (492us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.787245:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350794e300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993095296:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.787252:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44249 00000100:00100000:0.0:1783520935.787260:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35837 00000100:00100000:1.0:1783520935.787265:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515346d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+179:3627:x1870152993096320:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.787266:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02558b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.787281:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00002000:00080000:1.0:1783520935.787282:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1486 count 1 00010000:00010000:2.0:1783520935.787284:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02558b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.787294:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02558b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.787298:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0255ae lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5ce:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.787304:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02558b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.787310:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515347480/2, rc 0, transno 4294975744 00010000:00010000:1.0:1783520935.787311:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c6aa0713/0xd5401cbabc022a18 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3108 expref: 821 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.787319:0:35276:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000038bf5646/0xd5401cbabc01f9ab lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02b6 expref: 821 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.787321:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02558b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.787326:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515347480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993093760:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.787347:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x5ce:0x0].0x0 since=36s v=(0 0) 00010000:00010000:1.0:1783520935.787351:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ce:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.787353:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0255ae lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.787354:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502250000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993095168:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.787356:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.787394:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c6aa0713/0xd5401cbabc022a18 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3108 expref: 820 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.787396:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025584 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.787404:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:1.0:1783520935.787407:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c6aa0713/0xd5401cbabc022a18 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3108 expref: 820 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.787416:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025584 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.787417:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502252300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993096576:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.787427:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9f800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993096704:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.787432:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025584 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.787442:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025584 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.787450:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025584 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.787460:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.787467:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:2.0:1783520935.787468:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+180:3627:x1870152993095040:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2246us (2325us total) trans 4294975745 rc 0/0 00000100:00100000:2.0:1783520935.787478:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44244 00000100:00100000:1.0:1783520935.787511:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872f00 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.787513:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993096576 00000100:00100000:2.0:1783520935.787519:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993096576 00010000:00010000:1.0:1783520935.787527:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1783520935.787533:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02558b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.787542:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.787546:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02558b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.787550:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44250 00010000:00010000:1.0:1783520935.787559:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02558b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.787564:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502252680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+181:3626:x1870152993096576:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.787574:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502250380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+181:3626:x1870152993094784:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2435us (2548us total) trans 4294975746 rc 0/0 00002000:00080000:2.0:1783520935.787583:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1487 count 1 00010000:00010000:1.0:1783520935.787585:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02558b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.787585:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44243 00010000:00010000:1.0:1783520935.787596:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02558b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.787598:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0255b5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5cf:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.787608:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.787616:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025584 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.787621:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.787622:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006c8ecaa1/0xd5401cbabc01f9d5 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0318 expref: 820 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.787625:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.787630:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.787635:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.787639:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5cf:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.787641:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025584 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.787644:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5cf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.787653:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0255b5 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.787657:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025584 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.787665:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.787677:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025584 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.787679:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006c8ecaa1/0xd5401cbabc01f9d5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0318 expref: 820 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.787692:0:6495:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002385920 00000100:00100000:1.0:1783520935.787698:0:6495:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002385920 00010000:00010000:0.0:1783520935.787702:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025584 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.787705:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006c8ecaa1/0xd5401cbabc01f9d5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0318 expref: 820 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.787728:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515344380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993096832:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.787734:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.787735:0:6495:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98405 00010000:00010000:0.0:1783520935.787739:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.787744:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.787747:0:6495:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502250e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:25342ed3-dd17-4439-8398-d8002e9f9f99+10:251796:x1870153002385920:12345-192.168.204.33@tcp:101:stat.0 00000100:00100000:3.0:1783520935.787751:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.787760:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895840 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.787761:0:6495:0:(ldlm_lockd.c:1265:ldlm_handle_enqueue()) ### server-side enqueue handler START 00000100:00100000:0.0:1783520935.787773:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993095680:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.787774:0:6495:0:(ldlm_lockd.c:1343:ldlm_handle_enqueue()) ### server-side enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7600/0xd5401cbabc0255bc lrc: 2/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT gid 0 flags: 0x40000000000000 nid: local remote: 0xc275876b859f34c7 expref: -99 pid: 6495 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520935.787792:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e1c6300/2, rc 0, transno 4294975745 00010000:00010000:0.0:1783520935.787794:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025592 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.787809:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993095040:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.787836:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002386048 00000100:00100000:2.0:1783520935.787840:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002386048 00000100:00100000:0.0:1783520935.787847:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.787861:0:6495:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6c00/0xd5401cbabc0255c3 lrc: 3/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x40000000000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.787862:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517247800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993096960:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.787862:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004a29fb1b/0xd5401cbabc01f9b2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02d2 expref: 821 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.787881:0:6495:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6c00/0xd5401cbabc0255c3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT flags: 0x50210000000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.787890:0:6495:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6c00/0xd5401cbabc0255c3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT flags: 0x40210000000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.787896:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002386176 00000100:00100000:3.0:1783520935.787898:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.787901:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002386176 00010000:00010000:2.0:1783520935.787919:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000077791a40/0xd5401cbabc01f9b9 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02e0 expref: 822 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.787924:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502253480/2, rc 0, transno 4294975746 00000004:00200000:1.0:1783520935.787926:0:6495:0:(mdt_handler.c:909:mdt_pack_attr2body()) [0x200000007:0x1:0x0]: returning size 69632 00000100:00100000:0.0:1783520935.787931:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.787933:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502253480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993094784:0@lo:6:osp-syn-1-0.0 00000004:00200000:1.0:1783520935.787934:0:6495:0:(mdt_handler.c:1574:mdt_getattr_internal()) dirent count 0 stripe count 1 MDT count 2 00000004:00010000:1.0:1783520935.787958:0:6495:0:(mdt_handler.c:2507:mdt_getattr_name_lock()) ### Returning lock to client ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6c00/0xd5401cbabc0255c3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT flags: 0x40210000000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.787962:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002386304 00000100:00100000:2.0:1783520935.787968:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35838 00000100:00100000:0.0:1783520935.787969:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002386304 00000100:00100000:2.0:1783520935.787981:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502250000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+823:2388:x1870153002386048:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.787987:0:6495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa7600/0xd5401cbabc0255bc lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x44000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34c7 expref: 12 pid: 6495 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.787988:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b59af2c4/0xd5401cbabc01f9c0 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02ee expref: 823 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.787992:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.788002:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004a29fb1b/0xd5401cbabc01f9b2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ca:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02d2 expref: 823 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.788023:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004a29fb1b/0xd5401cbabc01f9b2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5ca:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02d2 expref: 823 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.788023:0:6495:0:(ldlm_lockd.c:1513:ldlm_handle_enqueue()) ### server-side enqueue handler, sending reply (err=0, rc=0) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6c00/0xd5401cbabc0255c3 lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x40200000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34c7 expref: 11 pid: 6495 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.788027:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35839 00000100:00100000:0.0:1783520935.788040:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+823:2388:x1870153002386176:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520935.788045:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35840 00010000:00010000:1.0:1783520935.788050:0:6495:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x0 (ffff943504b72c00) 00010000:00010000:0.0:1783520935.788053:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.788054:0:6495:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 13 00010000:00010000:2.0:1783520935.788060:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004a29fb1b/0xd5401cbabc01f9b2 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5ca:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02d2 expref: 823 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.788060:0:6495:0:(ldlm_lockd.c:1592:ldlm_handle_enqueue()) ### server-side enqueue handler END (lock ffff943514fa6c00, rc 0) 00000100:00100000:3.0:1783520935.788061:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357d450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+823:2388:x1870153002386304:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.788062:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000077791a40/0xd5401cbabc01f9b9 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5cb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02e0 expref: 823 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.788076:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.788084:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b59af2c4/0xd5401cbabc01f9c0 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5cc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02ee expref: 823 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.788085:0:6495:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502250e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:25342ed3-dd17-4439-8398-d8002e9f9f99+11:251796:x1870153002385920:12345-192.168.204.33@tcp:101:stat.0 Request processed in 341us (421us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.788088:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ca:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.788090:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000077791a40/0xd5401cbabc01f9b9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5cb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02e0 expref: 822 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.788099:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ca:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.788104:0:6495:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98405 00010000:00010000:2.0:1783520935.788112:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025592 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.788119:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b59af2c4/0xd5401cbabc01f9c0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5cc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02ee expref: 822 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.788119:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5cb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.788128:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025592 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.788128:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5cb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.788135:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025599 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5cb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.788138:0:35282:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000077791a40/0xd5401cbabc01f9b9 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5cb:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02e0 expref: 822 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.788148:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5cc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.788157:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5cc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.788159:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025599 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5cb:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.788164:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9c000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993095808:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.788166:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0255a0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5cc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.788171:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c553480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993097088:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.788181:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025599 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.788181:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.788183:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0255a0 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5cc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.788201:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025599 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.788203:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+821:2388:x1870153002386176:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 166us (357us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.788209:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.788217:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.788220:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025599 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.788221:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35839 00000100:00100000:3.0:1783520935.788232:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357d450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+820:2388:x1870153002386304:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 175us (305us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.788235:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025599 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.788241:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35840 00000100:00100000:0.0:1783520935.788245:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993096960 00000100:00100000:0.0:1783520935.788249:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993096960 00010000:00010000:1.0:1783520935.788251:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025599 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.788272:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993097088 00010000:00010000:1.0:1783520935.788273:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.788275:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993097088 00010000:00010000:2.0:1783520935.788279:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.788286:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025599 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.788295:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7800 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520935.788299:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44251 00010000:00010000:2.0:1783520935.788300:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025599 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.788309:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517244e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+182:3626:x1870152993096960:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.788312:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025599 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.788322:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1488 count 1 00010000:00010000:2.0:1783520935.788327:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025599 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.788334:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc0255ca lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5d0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.788336:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc025599 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.788346:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44252 00010000:00010000:0.0:1783520935.788350:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000028758043/0xd5401cbabc01f9dc lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0326 expref: 819 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.788374:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515345500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+183:3627:x1870152993097088:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520935.788387:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1489 count 1 00010000:00010000:0.0:1783520935.788399:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d0:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.788402:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.788405:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc0255ca lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.788420:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000028758043/0xd5401cbabc01f9dc lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0326 expref: 819 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.788427:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000028758043/0xd5401cbabc01f9dc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0326 expref: 819 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.788430:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.788434:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.788434:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502253800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+183:3625:x1870152993095552:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2567us (2689us total) trans 4294975747 rc 0/0 00010000:00010000:2.0:1783520935.788440:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.788440:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993097216:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.788444:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0255d1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5d1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.788447:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44246 00010000:00010000:3.0:1783520935.788459:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000008c629e5/0xd5401cbabc01f9e3 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0334 expref: 819 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.788465:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d1:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.788467:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.788468:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.788472:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0255d1 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.788482:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470180 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.788484:0:178496:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b59af2c4/0xd5401cbabc01f9c0 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02ee expref: 819 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.788491:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000008c629e5/0xd5401cbabc01f9e3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0334 expref: 819 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.788513:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9c380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993096192:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.788515:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000008c629e5/0xd5401cbabc01f9e3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0334 expref: 818 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.788515:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502250000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+818:2388:x1870153002386048:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 519us (752us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.788515:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025592 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.788525:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0255a0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.788539:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35838 00010000:00010000:0.0:1783520935.788541:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025592 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.788547:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c552d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993097344:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.788550:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0255a0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.788559:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025592 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.788561:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0255a0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.788575:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967e40 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.788576:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025592 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.788579:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0255a0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.788591:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0255a0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.788591:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000004:00080000:3.0:1783520935.788598:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9f100/2, rc 0, transno 4294975747 00010000:00010000:0.0:1783520935.788599:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.788606:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025592 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.788611:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993095552:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.788614:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.788615:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.788618:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025592 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.788624:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025592 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.788631:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025592 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.788636:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025592 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.788648:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.788651:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0255a0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.788656:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.788659:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0255a0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.788661:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502250000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993097472:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.788670:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0255a0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.788681:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0255a0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.788687:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc0255a0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.788695:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.788705:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502252a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+183:3626:x1870152993095936:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2063us (2235us total) trans 4294975748 rc 0/0 00010000:00010000:0.0:1783520935.788707:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.788713:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.788714:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44247 00010000:00010000:0.0:1783520935.788718:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:2.0:1783520935.788731:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9d880/2, rc 0, transno 4294975748 00000100:00100000:1.0:1783520935.788732:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993097472 00000100:00100000:1.0:1783520935.788738:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993097472 00000100:00100000:2.0:1783520935.788749:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993095936:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.788773:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44253 00000100:00100000:3.0:1783520935.788781:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c553800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993097600:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.788787:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c553100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+184:3628:x1870152993097472:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.788802:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1490 count 1 00000100:00100000:0.0:1783520935.788811:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+184:3627:x1870152993095424:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3036us (3169us total) trans 4294975749 rc 0/0 00000004:00080000:2.0:1783520935.788817:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e1c5500/2, rc 0, transno 4294975749 00000100:00100000:3.0:1783520935.788818:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.788818:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc0255d8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5d2:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.788827:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44245 00000100:00100000:2.0:1783520935.788834:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993095424:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.788840:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d28b53c0/0xd5401cbabc01f9ea lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0342 expref: 817 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.788847:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d2:0x0].0x0 since=37s v=(0 0) 00000100:00100000:0.0:1783520935.788849:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993097600 00010000:00010000:1.0:1783520935.788850:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.788855:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc0255d8 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.788856:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993097600 00010000:00010000:1.0:1783520935.788874:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d28b53c0/0xd5401cbabc01f9ea lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0342 expref: 817 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.788887:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d28b53c0/0xd5401cbabc01f9ea lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0342 expref: 817 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.788888:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44254 00000100:00100000:1.0:1783520935.788900:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9c380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993097728:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.788903:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350794e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+184:3625:x1870152993097600:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.788916:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1491 count 1 00010000:00010000:0.0:1783520935.788926:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0255df lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5d3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.788927:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6300 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.788939:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004b2a2908/0xd5401cbabc01f9f1 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0350 expref: 817 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.788943:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.788944:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d3:0x0].0x0 since=37s v=(0 0) 00000100:00100000:3.0:1783520935.788948:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:0.0:1783520935.788948:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.788954:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0255df lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.788958:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350794dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993097856:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.788973:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004b2a2908/0xd5401cbabc01f9f1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0350 expref: 817 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.788980:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004b2a2908/0xd5401cbabc01f9f1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0350 expref: 817 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.788988:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993097984:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.789015:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9d500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993096448:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.789016:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124706c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.789038:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0255a7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.789043:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.789066:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993097856 00000100:00100000:1.0:1783520935.789072:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993097856 00000100:00100000:1.0:1783520935.789106:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44255 00000100:00100000:2.0:1783520935.789116:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9f800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993096704:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.789116:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350794f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+185:3625:x1870152993097856:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.789135:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1492 count 1 00010000:00010000:2.0:1783520935.789136:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0255ae lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.789148:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0255e6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5d4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.789161:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005cafb6a8/0xd5401cbabc01f9f8 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f035e expref: 817 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.789167:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d4:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.789169:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.789173:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0255e6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.789189:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005cafb6a8/0xd5401cbabc01f9f8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f035e expref: 817 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.789202:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005cafb6a8/0xd5401cbabc01f9f8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f035e expref: 817 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.789213:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9c000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993098112:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.789240:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6cc0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.789320:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515344380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993096832:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.789330:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0255b5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.789395:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.789416:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002386432 00000100:00100000:0.0:1783520935.789421:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002386432 00000100:00100000:2.0:1783520935.789445:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.789446:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000020772469/0xd5401cbabc01f9c7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02fc expref: 818 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.789472:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002386688 00000100:00100000:0.0:1783520935.789476:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35841 00000100:00100000:1.0:1783520935.789479:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002386688 00000100:00100000:0.0:1783520935.789486:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e0f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+819:2387:x1870153002386432:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.789506:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.789515:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006c8ecaa1/0xd5401cbabc01f9d5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0318 expref: 819 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.789515:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000020772469/0xd5401cbabc01f9c7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5cd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02fc expref: 819 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.789543:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000020772469/0xd5401cbabc01f9c7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5cd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02fc expref: 819 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.789558:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35842 00010000:00010000:0.0:1783520935.789561:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000020772469/0xd5401cbabc01f9c7 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5cd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f02fc expref: 819 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.789572:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514f99c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+819:2387:x1870153002386688:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.789587:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.789588:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5cd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.789594:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5cd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.789595:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006c8ecaa1/0xd5401cbabc01f9d5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5cf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0318 expref: 818 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.789601:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0255a7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.789615:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006c8ecaa1/0xd5401cbabc01f9d5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5cf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0318 expref: 818 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.789616:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0255a7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.789628:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.789646:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006c8ecaa1/0xd5401cbabc01f9d5 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5cf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0318 expref: 818 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.789652:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.789655:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002386560 00010000:00010000:1.0:1783520935.789659:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5cf:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.789660:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002386560 00010000:00010000:1.0:1783520935.789667:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5cf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.789672:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0255b5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.789679:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c6aa0713/0xd5401cbabc022a18 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3108 expref: 818 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.789682:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0255b5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.789683:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e0f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+818:2387:x1870153002386432:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 199us (290us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.789700:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35841 00000100:00100000:0.0:1783520935.789716:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35843 00010000:00010000:1.0:1783520935.789717:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.789726:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514f9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+817:2390:x1870153002386560:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.789736:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.789742:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c6aa0713/0xd5401cbabc022a18 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5ce:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3108 expref: 817 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.789752:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514f99c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+817:2387:x1870153002386688:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 184us (311us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.789756:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c6aa0713/0xd5401cbabc022a18 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5ce:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3108 expref: 817 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.789766:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35842 00010000:00010000:0.0:1783520935.789779:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c6aa0713/0xd5401cbabc022a18 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5ce:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3108 expref: 816 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.789792:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0255b5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.789792:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ce:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.789795:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ce:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.789805:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0255ae lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.789807:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0255b5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.789826:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0255b5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.789827:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0255ae lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.789848:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0255b5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.789856:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.789866:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.789873:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0255ae lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.789883:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0255ae lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.789885:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514f9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+815:2390:x1870153002386560:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 162us (262us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.789903:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0255ae lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.789903:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35843 00010000:00010000:2.0:1783520935.789913:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0255ae lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.789924:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520935.789929:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.789932:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0255b5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.789942:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.789943:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520935.789944:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502252680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+186:3626:x1870152993096576:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2383us (2484us total) trans 4294975750 rc 0/0 00010000:00010000:0.0:1783520935.789947:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0255b5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.789951:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0255ae lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.789957:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44250 00000100:00100000:3.0:1783520935.789966:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.789973:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0255b5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.789974:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.789980:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0255ae lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.789989:0:35280:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002386816 00010000:00010000:0.0:1783520935.789993:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0255b5 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.789994:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0255ae lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.789994:0:35280:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002386816 00000100:00100000:1.0:1783520935.790020:0:35280:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98406 00010000:00010000:2.0:1783520935.790024:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0255ae lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.790030:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0255b5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.790032:0:35280:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350794ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_011:25342ed3-dd17-4439-8398-d8002e9f9f99+11:251796:x1870153002386816:12345-192.168.204.33@tcp:101:stat.0 00010000:00010000:1.0:1783520935.790046:0:35280:0:(ldlm_lockd.c:1265:ldlm_handle_enqueue()) ### server-side enqueue handler START 00010000:00010000:2.0:1783520935.790047:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0255ae lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.790057:0:35280:0:(ldlm_lockd.c:1343:ldlm_handle_enqueue()) ### server-side enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6a00/0xd5401cbabc0255ed lrc: 2/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT gid 0 flags: 0x40000000000000 nid: local remote: 0xc275876b859f34ce expref: -99 pid: 35280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.790058:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.790062:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.790067:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.790068:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.790072:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.790076:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.790082:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0255a7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.790092:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0255a7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.790100:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0255a7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.790107:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0255a7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.790116:0:35280:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc0255f4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x40000000000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.790122:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.790125:0:35280:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc0255f4 lrc: 3/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210000000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.790131:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.790133:0:35280:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc0255f4 lrc: 3/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.790138:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0255a7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.790145:0:35280:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6600/0xd5401cbabc0255fb lrc: 3/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x40000000000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.790146:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.790153:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0255a7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.790155:0:35280:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6600/0xd5401cbabc0255fb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.790169:0:35280:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6600/0xd5401cbabc0255fb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210000000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.790176:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0255a7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.790178:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515346d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+186:3627:x1870152993096320:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2915us (3023us total) trans 4294975751 rc 0/0 00000100:00100000:2.0:1783520935.790194:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44249 00000004:00080000:3.0:1783520935.790197:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350794c000/2, rc 0, transno 4294975751 00010000:00010000:0.0:1783520935.790198:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0255a7 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.790206:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0255a7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.790208:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350794c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993096320:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.790221:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.790223:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.790224:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.790251:0:35280:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6e00/0xd5401cbabc025602 lrc: 3/1,0 mode: --/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x0/0x0 rrc: 6 type: IBT flags: 0x40000000000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.790256:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:1.0:1783520935.790259:0:35280:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6e00/0xd5401cbabc025602 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x5b/0x0 rrc: 6 type: IBT flags: 0x50210000000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.790267:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350794d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993098240:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.790270:0:35280:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6e00/0xd5401cbabc025602 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x5b/0x0 rrc: 6 type: IBT flags: 0x40210000000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.790292:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00200000:1.0:1783520935.790298:0:35280:0:(mdt_handler.c:909:mdt_pack_attr2body()) [0x200000402:0x72a:0x0]: returning size 262144 00000004:00200000:1.0:1783520935.790302:0:35280:0:(mdt_handler.c:1574:mdt_getattr_internal()) dirent count 0 stripe count 1 MDT count 2 00000100:00100000:0.0:1783520935.790305:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502253100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+186:3627:x1870152993096064:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3333us (3523us total) trans 4294975752 rc 0/0 00000004:00010000:1.0:1783520935.790315:0:35280:0:(mdt_handler.c:2507:mdt_getattr_name_lock()) ### Returning lock to client ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6e00/0xd5401cbabc025602 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x5b/0x0 rrc: 6 type: IBT flags: 0x40210000000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.790315:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44248 00000004:00080000:2.0:1783520935.790322:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502253b80/2, rc 0, transno 4294975752 00010000:00010000:1.0:1783520935.790326:0:35280:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6e00/0xd5401cbabc025602 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT flags: 0x50210000000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.790333:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993098240 00010000:00010000:1.0:1783520935.790334:0:35280:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc0255f4 lrc: 4/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.790337:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502253b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993096064:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.790340:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993098240 00010000:00010000:1.0:1783520935.790348:0:35280:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc0255f4 lrc: 4/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210400000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.790354:0:35280:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc0255f4 lrc: 3/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210400000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.790388:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44256 00010000:00010000:1.0:1783520935.790389:0:35280:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc0255f4 lrc: 4/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210400000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.790395:0:35280:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6800 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:1.0:1783520935.790399:0:35280:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc0255f4 lrc: 4/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210400000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.790400:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350794df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+186:3628:x1870152993098240:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.790411:0:35280:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc0255f4 lrc: 5/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40218400000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.790418:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1493 count 1 00010000:00010000:1.0:1783520935.790419:0:35280:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x0 (ffff943504b72c00) 00010000:00010000:1.0:1783520935.790424:0:35280:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:1.0:1783520935.790427:0:35280:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc0255f4 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x44a19400000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.790428:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515344380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993098368:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.790437:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025609 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5d5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.790439:0:35280:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6800/0xd5401cbabc0255f4 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x44a19400000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.790458:0:35280:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6600/0xd5401cbabc0255fb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.790460:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c5ebd3cc/0xd5401cbabc01f9ff lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f036c expref: 814 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.790468:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.790471:0:35280:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6600/0xd5401cbabc0255fb lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210400000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.790484:0:35280:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6600/0xd5401cbabc0255fb lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.790509:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d5:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.790511:0:35280:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6600 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:0.0:1783520935.790514:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.790516:0:35280:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6600/0xd5401cbabc0255fb lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.790521:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025609 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.790523:0:35280:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6600/0xd5401cbabc0255fb lrc: 4/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40218400000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.790539:0:35280:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x5ba6781 (ffff943507a8d500) 00010000:00010000:1.0:1783520935.790541:0:35280:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:1.0:1783520935.790543:0:35280:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6600/0xd5401cbabc0255fb lrc: 2/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.790548:0:35280:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6600/0xd5401cbabc0255fb lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.790554:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c5ebd3cc/0xd5401cbabc01f9ff lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f036c expref: 814 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.790566:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c5ebd3cc/0xd5401cbabc01f9ff lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f036c expref: 814 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.790571:0:35280:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6a00/0xd5401cbabc0255ed lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x44000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34ce expref: 13 pid: 35280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.790589:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993098496:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.790602:0:35280:0:(ldlm_lockd.c:1513:ldlm_handle_enqueue()) ### server-side enqueue handler, sending reply (err=0, rc=0) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6e00/0xd5401cbabc025602 lrc: 3/0,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x40200000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34ce expref: 12 pid: 35280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.790609:0:35280:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x72a:0x0].0x0 (ffff943515473e00) 00010000:00010000:1.0:1783520935.790612:0:35280:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 1b 00010000:00010000:1.0:1783520935.790615:0:35280:0:(ldlm_lockd.c:1592:ldlm_handle_enqueue()) ### server-side enqueue handler END (lock ffff943514fa6e00, rc 0) 00000100:00100000:0.0:1783520935.790634:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470d80 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.790644:0:35280:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350794ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_011:25342ed3-dd17-4439-8398-d8002e9f9f99+12:251796:x1870153002386816:12345-192.168.204.33@tcp:101:stat.0 Request processed in 614us (725us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.790653:0:35280:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98406 00000100:00100000:0.0:1783520935.790653:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993098368 00000100:00100000:0.0:1783520935.790659:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993098368 00000100:00100000:0.0:1783520935.790691:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44257 00000100:00100000:0.0:1783520935.790701:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cc7c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+187:3628:x1870152993098368:12345-0@lo:6:osp-syn-1-0.0 00000004:00080000:1.0:1783520935.790702:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502252300/2, rc 0, transno 4294975750 00002000:00080000:0.0:1783520935.790717:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1494 count 1 00000100:00100000:1.0:1783520935.790719:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502252300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993096576:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.790732:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025610 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5d6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.790745:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ea31747f/0xd5401cbabc022907 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ff7 expref: 814 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.790758:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x5d6:0x0].0x0 since=36s v=(0 0) 00010000:00010000:0.0:1783520935.790761:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d6:0x0].0x0 since=0s v=(1 0) 00000100:00100000:3.0:1783520935.790765:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943611039500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993098624:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.790766:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025610 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.790777:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.790786:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ea31747f/0xd5401cbabc022907 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ff7 expref: 814 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.790798:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ea31747f/0xd5401cbabc022907 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ff7 expref: 814 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.790809:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad1180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993098752:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.790829:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9c380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993097728:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.790841:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.790845:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470b40 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.790847:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc0255d8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.790878:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993097216:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.790889:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.790896:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc0255ca lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.790915:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993098624 00000100:00100000:1.0:1783520935.790920:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993098624 00000100:00100000:0.0:1783520935.790940:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993097984:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.790960:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44258 00010000:00010000:0.0:1783520935.790960:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0255df lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.790964:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002386944 00000100:00100000:3.0:1783520935.790969:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002386944 00000100:00100000:2.0:1783520935.790970:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.790973:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943611038e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+188:3625:x1870152993098624:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.790990:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000028758043/0xd5401cbabc01f9dc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0326 expref: 815 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.790991:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1495 count 1 00000100:00100000:2.0:1783520935.791004:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.791008:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025617 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5d7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.791018:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002387072 00010000:00010000:1.0:1783520935.791020:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007dc031da/0xd5401cbabc01fa0d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0388 expref: 815 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.791028:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002387072 00010000:00010000:1.0:1783520935.791031:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d7:0x0].0x0 since=37s v=(0 0) 00000100:00100000:0.0:1783520935.791031:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002387328 00000100:00100000:0.0:1783520935.791036:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002387328 00010000:00010000:1.0:1783520935.791038:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d7:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.791043:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.791048:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000008c629e5/0xd5401cbabc01f9e3 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0334 expref: 817 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.791048:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025617 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.791053:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004b2a2908/0xd5401cbabc01f9f1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0350 expref: 817 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.791067:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002387200 00000100:00100000:2.0:1783520935.791073:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002387200 00000100:00100000:0.0:1783520935.791085:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002387456 00000100:00100000:0.0:1783520935.791090:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002387456 00000100:00100000:3.0:1783520935.791092:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35844 00010000:00010000:1.0:1783520935.791092:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007dc031da/0xd5401cbabc01fa0d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0388 expref: 818 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.791097:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d28b53c0/0xd5401cbabc01f9ea lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0342 expref: 819 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.791105:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cc7ea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+819:2387:x1870153002386944:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.791105:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005cafb6a8/0xd5401cbabc01f9f8 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f035e expref: 819 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.791114:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007dc031da/0xd5401cbabc01fa0d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0388 expref: 819 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.791118:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.791127:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000028758043/0xd5401cbabc01f9dc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5d0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0326 expref: 819 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.791136:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9c380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993098880:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.791139:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35845 00000100:00100000:0.0:1783520935.791149:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35846 00000100:00100000:2.0:1783520935.791153:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505876300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+819:2387:x1870153002387072:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.791163:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505877100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+819:2388:x1870153002387328:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.791164:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.791174:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000008c629e5/0xd5401cbabc01f9e3 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5d1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0334 expref: 819 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.791175:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6300 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.791176:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.791185:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004b2a2908/0xd5401cbabc01f9f1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5d3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0350 expref: 819 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.791199:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000008c629e5/0xd5401cbabc01f9e3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5d1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0334 expref: 819 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.791204:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000028758043/0xd5401cbabc01f9dc lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5d0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0326 expref: 819 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.791204:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004b2a2908/0xd5401cbabc01f9f1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5d3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0350 expref: 819 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.791205:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9c000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993098112:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.791226:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0255e6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.791228:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.791234:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000028758043/0xd5401cbabc01f9dc lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5d0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0326 expref: 819 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.791234:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004b2a2908/0xd5401cbabc01f9f1 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5d3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0350 expref: 819 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.791238:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.791248:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0255d1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.791258:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.791265:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.791271:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0255d1 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5d1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.791272:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0255df lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.791273:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.791280:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.791288:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc0255ca lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.791292:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0255df lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.791309:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc0255ca lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.791319:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.791325:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.791326:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.791342:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc0255ca lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.791344:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505876300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+817:2387:x1870153002387072:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 194us (503us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.791355:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cc7ea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+817:2387:x1870153002386944:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 249us (576us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.791355:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505877100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+817:2388:x1870153002387328:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 192us (387us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.791379:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc0255ca lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.791387:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35845 00000100:00100000:3.0:1783520935.791400:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35844 00010000:00010000:1.0:1783520935.791405:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc0255ca lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.791416:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35847 00010000:00010000:1.0:1783520935.791424:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc0255ca lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.791430:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35846 00000100:00100000:2.0:1783520935.791432:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638586850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+815:2388:x1870153002387200:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520935.791441:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35848 00010000:00010000:2.0:1783520935.791447:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520935.791453:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505876680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+814:2387:x1870153002387456:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.791456:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d28b53c0/0xd5401cbabc01f9ea lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5d2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0342 expref: 814 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.791457:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.791464:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.791465:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.791473:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005cafb6a8/0xd5401cbabc01f9f8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5d4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f035e expref: 814 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.791473:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc0255ca lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.791477:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d28b53c0/0xd5401cbabc01f9ea lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5d2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0342 expref: 814 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.791489:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.791507:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005cafb6a8/0xd5401cbabc01f9f8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5d4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f035e expref: 814 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.791507:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc0255ca lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.791519:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc0255ca lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.791521:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d28b53c0/0xd5401cbabc01f9ea lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5d2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0342 expref: 814 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.791526:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005cafb6a8/0xd5401cbabc01f9f8 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5d4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f035e expref: 814 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.791540:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc0255ca lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.791553:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.791553:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc0255ca lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.791556:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d4:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.791558:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517244e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+189:3626:x1870152993096960:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3249us (3657us total) trans 4294975753 rc 0/0 00010000:00010000:2.0:1783520935.791563:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.791564:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d4:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.791577:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44251 00010000:00010000:3.0:1783520935.791578:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0255e6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.791578:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc0255d8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.791582:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.791585:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.791588:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.791603:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc0255d8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.791604:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0255e6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.791613:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517247800/2, rc 0, transno 4294975753 00000100:00100000:1.0:1783520935.791627:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517247800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993096960:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.791640:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.791651:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0255e6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.791653:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.791663:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0255e6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.791673:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0255e6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.791681:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638586850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+812:2388:x1870153002387200:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 251us (674us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.791681:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993099008:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.791683:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0255e6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.791700:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35847 00010000:00010000:0.0:1783520935.791704:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.791712:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.791716:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0255e6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.791723:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6000 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520935.791726:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.791727:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0255e6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.791735:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0255e6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.791744:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0255e6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.791752:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0255e6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.791772:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.791775:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.791779:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.791782:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0255df lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.791792:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0255df lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.791801:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0255df lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.791808:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0255df lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.791826:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.791882:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350794f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+190:3625:x1870152993097856:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2768us (2840us total) trans 4294975754 rc 0/0 00000100:00100000:0.0:1783520935.791890:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44255 00000100:00100000:0.0:1783520935.791904:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993099008 00000100:00100000:0.0:1783520935.791907:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993099008 00000100:00100000:1.0:1783520935.791921:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350794e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+190:3625:x1870152993097600:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3019us (3103us total) trans 4294975755 rc 0/0 00000100:00100000:2.0:1783520935.791922:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.791928:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44259 00000100:00100000:1.0:1783520935.791937:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44254 00000100:00100000:0.0:1783520935.791942:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+190:3626:x1870152993099008:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.791957:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1496 count 1 00010000:00010000:0.0:1783520935.791971:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02561e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5d8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.791976:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350794dc00/2, rc 0, transno 4294975754 00010000:00010000:0.0:1783520935.791984:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000002c11609/0xd5401cbabc01fa14 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0396 expref: 811 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.791987:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.791992:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350794dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993097856:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.791999:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d8:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.792004:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.792008:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02561e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.792013:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c553800/2, rc 0, transno 4294975755 00000100:00100000:1.0:1783520935.792020:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c553800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993097600:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.792029:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.792030:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000002c11609/0xd5401cbabc01fa14 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0396 expref: 811 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.792045:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000002c11609/0xd5401cbabc01fa14 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0396 expref: 811 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.792054:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520935.792059:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993099136:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.792061:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0255df lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.792064:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002387584 00000100:00100000:3.0:1783520935.792069:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002387584 00010000:00010000:2.0:1783520935.792075:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfa600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.792080:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0255df lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.792089:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c5ebd3cc/0xd5401cbabc01f9ff lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f036c expref: 812 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.792092:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0255df lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.792097:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993099264:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.792102:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124706c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.792119:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0255df lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.792120:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002387712 00000100:00100000:0.0:1783520935.792120:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002387840 00000100:00100000:3.0:1783520935.792125:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002387712 00000100:00100000:0.0:1783520935.792125:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002387840 00010000:00010000:2.0:1783520935.792132:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0255df lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.792141:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.792144:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ea31747f/0xd5401cbabc022907 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x5d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ff7 expref: 813 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.792155:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993099264 00000100:00100000:0.0:1783520935.792157:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993099264 00010000:00010000:2.0:1783520935.792162:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.792165:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35849 00010000:00010000:2.0:1783520935.792168:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.792174:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.792176:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1783520935.792178:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505876300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+813:2390:x1870153002387584:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.792181:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 44260 00010000:00010000:3.0:1783520935.792188:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.792188:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993099392:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.792189:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505875f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:25342ed3-dd17-4439-8398-d8002e9f9f99+31:2387:x1870153002387840:12345-192.168.204.33@tcp:101:stat.0 00010000:00010000:3.0:1783520935.792195:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c5ebd3cc/0xd5401cbabc01f9ff lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5d5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f036c expref: 813 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.792202:0:35274:0:(ldlm_lockd.c:1265:ldlm_handle_enqueue()) ### server-side enqueue handler START 00000100:00100000:2.0:1783520935.792205:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35850 00010000:00010000:0.0:1783520935.792213:0:35274:0:(ldlm_lockd.c:1343:ldlm_handle_enqueue()) ### server-side enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025625 lrc: 2/0,0 mode: --/PR res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0xc275876b859f34d5 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.792218:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c5ebd3cc/0xd5401cbabc01f9ff lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5d5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f036c expref: 813 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.792224:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505877100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+813:2388:x1870153002387712:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.792228:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.792242:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.792244:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d5:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.792246:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993099520:192.168.204.33@tcp:106: 00010000:00010000:3.0:1783520935.792249:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.792252:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ea31747f/0xd5401cbabc022907 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x5d6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ff7 expref: 813 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.792255:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025609 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.792259:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993099392 00000100:00100000:1.0:1783520935.792265:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993099392 00000100:00100000:0.0:1783520935.792273:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470180 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.792277:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025609 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5d5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.792280:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ea31747f/0xd5401cbabc022907 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5d6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ff7 expref: 813 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.792296:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44261 00000100:00100000:0.0:1783520935.792298:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad1180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993098752:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.792300:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.792312:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+191:3627:x1870152993099264:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.792313:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ea31747f/0xd5401cbabc022907 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5d6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ff7 expref: 813 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.792315:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025610 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5d6:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.792322:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505876300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+813:2390:x1870153002387584:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 147us (402us total) trans 0 rc 0/0 00002000:00080000:1.0:1783520935.792333:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1497 count 1 00000100:00100000:3.0:1783520935.792337:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35849 00010000:00010000:2.0:1783520935.792345:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.792351:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d6:0x0].0x0 since=0s v=(1 0) 00000100:00100000:0.0:1783520935.792351:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44262 00010000:00010000:1.0:1783520935.792352:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02562c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5d9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.792387:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025610 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.792396:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c2c434c5/0xd5401cbabc01fa1b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f03a4 expref: 811 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.792399:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025610 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.792404:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+191:3627:x1870152993099392:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.792424:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1498 count 1 00000100:00100000:3.0:1783520935.792434:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505876680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+811:2387:x1870153002387456:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 982us (1393us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.792434:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d9:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.792440:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.792440:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc025633 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5da:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.792441:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d9:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.792449:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35848 00010000:00010000:1.0:1783520935.792451:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02562c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.792469:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505877100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+810:2388:x1870153002387712:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 250us (484us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.792476:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009ebdd8fa/0xd5401cbabc01fa22 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f03b2 expref: 810 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.792483:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35850 00010000:00010000:3.0:1783520935.792484:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000008c629e5/0xd5401cbabc01f9e3 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0334 expref: 810 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.792489:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c2c434c5/0xd5401cbabc01fa1b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03a4 expref: 810 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.792491:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5da:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.792511:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5da:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.792520:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc025633 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.792527:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c2c434c5/0xd5401cbabc01fa1b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03a4 expref: 808 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.792543:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9ce00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993099648:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.792551:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009ebdd8fa/0xd5401cbabc01fa22 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03b2 expref: 808 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.792563:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009ebdd8fa/0xd5401cbabc01fa22 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03b2 expref: 808 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.792575:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad1180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993099776:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.792592:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c552d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993097344:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.792594:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6840 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.792606:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470b40 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.792616:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.792627:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025610 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.792629:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc0255d8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.792643:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0255d1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.792649:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025610 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.792654:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc0255d8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.792662:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025610 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.792675:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0255d1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.792681:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc0255d8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.792687:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0255d1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.792687:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025610 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.792704:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002387968 00010000:00010000:1.0:1783520935.792704:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc0255d8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.792705:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0255d1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.792709:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002387968 00010000:00010000:0.0:1783520935.792710:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.792722:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.792725:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007dc031da/0xd5401cbabc01fa0d lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0388 expref: 809 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.792728:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0255d1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.792729:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025610 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.792752:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfbc00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.792756:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025610 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.792761:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:2.0:1783520935.792764:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35851 00010000:00010000:0.0:1783520935.792764:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025610 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.792774:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e0f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+809:2388:x1870153002387968:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.792782:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.792782:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025610 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.792788:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007dc031da/0xd5401cbabc01fa0d lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5d7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0388 expref: 809 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.792796:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:0.0:1783520935.792804:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc025610 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.792808:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007dc031da/0xd5401cbabc01fa0d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5d7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0388 expref: 809 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.792828:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.792832:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.792832:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520935.792841:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025617 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.792841:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc0255d8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.792857:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025617 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5d7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.792863:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c553100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+193:3628:x1870152993097472:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4079us (4165us total) trans 4294975756 rc 0/0 00010000:00010000:0.0:1783520935.792880:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7400 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:3.0:1783520935.792881:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515345500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+193:3627:x1870152993097088:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4525us (4669us total) trans 4294975757 rc 0/0 00000100:00100000:1.0:1783520935.792885:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44253 00010000:00010000:0.0:1783520935.792887:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc0255d8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.792893:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44252 00000004:00080000:2.0:1783520935.792896:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502250000/2, rc 0, transno 4294975756 00010000:00010000:0.0:1783520935.792898:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc0255d8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.792912:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502250000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993097472:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.792912:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc0255d8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.792925:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc0255d8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.792932:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c553480/2, rc 0, transno 4294975757 00000100:00100000:3.0:1783520935.792939:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c553480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993097088:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.792939:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.792945:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.792948:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0255d1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.792956:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.792961:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0255d1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.792966:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e0f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+809:2388:x1870153002387968:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 191us (351us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.792972:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993100032:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.792980:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0255d1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.792981:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35851 00010000:00010000:0.0:1783520935.792996:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0255d1 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.793004:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0255d1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.793011:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.793026:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.793028:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.793031:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.793037:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.793038:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993100032 00010000:00010000:0.0:1783520935.793041:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.793042:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993100032 00000100:00100000:1.0:1783520935.793046:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1783520935.793054:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993099904:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.793073:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44263 00000100:00100000:1.0:1783520935.793084:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.793085:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+193:3626:x1870152993100032:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.793101:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993099904 00000100:00100000:2.0:1783520935.793105:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993099904 00000100:00100000:0.0:1783520935.793130:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cc7c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+194:3628:x1870152993098368:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2431us (2667us total) trans 4294975758 rc 0/0 00000100:00100000:2.0:1783520935.793134:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44264 00002000:00080000:3.0:1783520935.793140:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1500 count 1 00000100:00100000:0.0:1783520935.793145:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44257 00000100:00100000:2.0:1783520935.793146:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+194:3626:x1870152993099904:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.793151:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007dc031da/0xd5401cbabc01fa0d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0388 expref: 808 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.793152:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02563a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5dc:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.793161:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1499 count 1 00010000:00010000:2.0:1783520935.793174:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025641 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5db:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.793180:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000228f4885/0xd5401cbabc01fa30 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f03ce expref: 807 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.793180:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9c380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993098880:192.168.204.33@tcp:104: 00000004:00080000:0.0:1783520935.793181:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515344380/2, rc 0, transno 4294975758 00010000:00010000:2.0:1783520935.793188:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c7b6824a/0xd5401cbabc01fa29 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f03c0 expref: 807 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.793201:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515344380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993098368:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.793202:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025617 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.793204:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5dc:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.793208:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5db:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.793212:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5dc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.793217:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5db:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.793223:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02563a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.793225:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025641 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.793233:0:20415:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c5ebd3cc/0xd5401cbabc01f9ff lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f036c expref: 807 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.793251:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993098496:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.793260:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c7b6824a/0xd5401cbabc01fa29 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03c0 expref: 806 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.793265:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000228f4885/0xd5401cbabc01fa30 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03ce expref: 806 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.793266:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:0.0:1783520935.793267:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025609 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.793272:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c7b6824a/0xd5401cbabc01fa29 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03c0 expref: 806 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.793280:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993100160:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.793287:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025609 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.793290:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000228f4885/0xd5401cbabc01fa30 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03ce expref: 806 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.793296:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514f99c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993100288:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.793299:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025609 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.793310:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94361103b100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993100416:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.793314:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025609 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.793319:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.793327:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025609 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.793333:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872f00 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.793347:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.793354:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.793419:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025617 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.793422:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025609 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.793434:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967840 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.793445:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfa000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.793446:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025617 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.793449:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993100160 00010000:00010000:2.0:1783520935.793452:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025609 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.793455:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993100160 00010000:00010000:2.0:1783520935.793478:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025609 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.793480:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44265 00010000:00010000:1.0:1783520935.793482:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025617 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.793488:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350794df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+195:3628:x1870152993098240:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3089us (3193us total) trans 4294975759 rc 0/0 00000100:00100000:3.0:1783520935.793492:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+195:3628:x1870152993100160:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.793517:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025609 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.793517:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44256 00010000:00010000:1.0:1783520935.793519:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025617 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520935.793521:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1501 count 1 00010000:00010000:3.0:1783520935.793536:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025648 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5dd:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.793541:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc025609 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.793561:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a9f387e4/0xd5401cbabc01fa37 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f03dc expref: 806 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.793563:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350794d180/2, rc 0, transno 4294975759 00010000:00010000:2.0:1783520935.793567:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.793571:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.793574:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.793580:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350794d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993098240:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.793605:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.793627:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5dd:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.793627:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520935.793631:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993100544:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.793632:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5dd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.793633:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025617 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.793641:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025648 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.793643:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.793647:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025617 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.793656:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025617 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.793664:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a9f387e4/0xd5401cbabc01fa37 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03dc expref: 806 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.793669:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025617 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.793677:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.793678:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a9f387e4/0xd5401cbabc01fa37 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03dc expref: 806 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.793684:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025617 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.793691:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943611038a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993100672:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.793704:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.793707:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.793710:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.793729:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435129679c0 going to sleep for 11 seconds 00002000:00010000:0.0:1783520935.793735:0:35274:0:(ofd_lvb.c:260:ofd_lvbo_update()) res: [0x2c0000401:0x8f7:0x0] updating lvb size: 196608 -> 262144 00000100:00100000:2.0:1783520935.793740:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00002000:00010000:0.0:1783520935.793741:0:35274:0:(ofd_lvb.c:272:ofd_lvbo_update()) res: [0x2c0000401:0x8f7:0x0] updating lvb atime: 0 -> 1783520935 00000100:00100000:3.0:1783520935.793745:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993100544 00002000:00010000:0.0:1783520935.793747:0:35274:0:(ofd_lvb.c:284:ofd_lvbo_update()) res: [0x2c0000401:0x8f7:0x0] updating lvb blocks: 0 -> 1 00000100:00100000:3.0:1783520935.793748:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993100544 00000100:00100000:0.0:1783520935.793759:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993099520:192.168.204.33@tcp:106: 00010000:00010000:0.0:1783520935.793783:0:35274:0:(ldlm_lockd.c:1513:ldlm_handle_enqueue()) ### server-side enqueue handler, sending reply (err=301, rc=0) ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025625 lrc: 1/0,0 mode: --/PR res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34d5 expref: 32 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.793799:0:35274:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc025625 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34d5 expref: 32 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.793809:0:35274:0:(ldlm_lockd.c:1592:ldlm_handle_enqueue()) ### server-side enqueue handler END (lock ffff94351cbfa200, rc 0) 00000100:00100000:1.0:1783520935.793811:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943611038e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+196:3625:x1870152993098624:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2841us (2919us total) trans 4294975760 rc 0/0 00000100:00100000:1.0:1783520935.793820:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44258 00000100:00100000:0.0:1783520935.793837:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505875f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:25342ed3-dd17-4439-8398-d8002e9f9f99+31:2387:x1870153002387840:12345-192.168.204.33@tcp:101:stat.0 Request processed in 1649us (1814us total) trans 0 rc 301/301 00000100:00100000:0.0:1783520935.793845:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 44260 00000004:00080000:1.0:1783520935.793851:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943611039500/2, rc 0, transno 4294975760 00000100:00100000:1.0:1783520935.793866:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943611039500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993098624:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.793881:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44266 00000100:00100000:0.0:1783520935.793885:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad1180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993099776:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.793890:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+195:3628:x1870152993100544:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.793900:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc025633 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.793911:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.793915:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1783520935.793926:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993100800:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.793935:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002388096 00000100:00100000:2.0:1783520935.793938:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002388096 00000100:00100000:0.0:1783520935.793941:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993099136:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.793955:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02561e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5d8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.793956:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000002c11609/0xd5401cbabc01fa14 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5d8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0396 expref: 807 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.793959:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00080000:3.0:1783520935.793963:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1502 count 1 00010000:00010000:3.0:1783520935.793978:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02564f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5de:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.793995:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993100800 00000100:00100000:0.0:1783520935.793999:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993100800 00000100:00100000:2.0:1783520935.794002:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002388224 00000100:00100000:2.0:1783520935.794006:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002388224 00000100:00100000:1.0:1783520935.794012:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9ce00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993099648:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.794019:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009ebdd8fa/0xd5401cbabc01fa22 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03b2 expref: 808 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.794031:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02562c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.794032:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44267 00000100:00100000:2.0:1783520935.794042:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35852 00000100:00100000:0.0:1783520935.794047:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+196:3626:x1870152993100800:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.794055:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ac01b8e6/0xd5401cbabc022b61 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3251 expref: 808 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.794057:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514f9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+808:2387:x1870153002388096:12345-192.168.204.33@tcp:103:ldlm_bl.0 00002000:00080000:0.0:1783520935.794065:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1503 count 1 00010000:00010000:2.0:1783520935.794071:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.794077:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025656 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5df:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.794079:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000002c11609/0xd5401cbabc01fa14 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5d8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0396 expref: 808 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.794080:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x5de:0x0].0x0 since=36s v=(0 0) 00010000:00010000:3.0:1783520935.794087:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5de:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520935.794095:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02564f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.794098:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005167c805/0xd5401cbabc01fa45 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f03f8 expref: 808 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.794100:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000002c11609/0xd5401cbabc01fa14 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5d8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0396 expref: 808 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.794115:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5df:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.794120:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5df:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.794125:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ac01b8e6/0xd5401cbabc022b61 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3251 expref: 808 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.794125:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025656 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.794127:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000002c11609/0xd5401cbabc01fa14 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5d8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0396 expref: 808 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.794146:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ac01b8e6/0xd5401cbabc022b61 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3251 expref: 808 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.794152:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.794152:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005167c805/0xd5401cbabc01fa45 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03f8 expref: 807 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.794161:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c552d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993100928:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.794161:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.794169:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005167c805/0xd5401cbabc01fa45 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03f8 expref: 807 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.794171:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02561e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.794186:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967e40 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.794191:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993101056:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.794194:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02561e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.794206:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35853 00000100:00100000:0.0:1783520935.794211:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124706c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.794217:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cc7ea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+807:2388:x1870153002388224:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.794226:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.794232:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009ebdd8fa/0xd5401cbabc01fa22 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5da:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03b2 expref: 807 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.794234:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.794246:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009ebdd8fa/0xd5401cbabc01fa22 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5da:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03b2 expref: 807 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.794260:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514f9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+807:2387:x1870153002388096:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 206us (522us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.794266:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02561e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.794270:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35852 00010000:00010000:1.0:1783520935.794282:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02561e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.794292:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02561e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.794300:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02561e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.794318:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.794321:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009ebdd8fa/0xd5401cbabc01fa22 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5da:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03b2 expref: 806 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.794333:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.794338:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.794339:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5da:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.794342:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5da:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.794344:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02561e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.794347:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc025633 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.794352:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.794356:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc025633 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.794374:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02561e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.794384:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02561e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.794397:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.794397:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02561e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.794404:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02561e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.794419:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cc7ea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+805:2388:x1870153002388224:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 203us (511us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.794426:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35853 00010000:00010000:0.0:1783520935.794437:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.794440:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.794441:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002388352 00010000:00010000:0.0:1783520935.794442:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.794445:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002388352 00010000:00010000:0.0:1783520935.794451:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc025633 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.794459:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc025633 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.794461:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+197:3626:x1870152993099008:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2520us (2736us total) trans 4294975761 rc 0/0 00010000:00010000:3.0:1783520935.794462:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c2c434c5/0xd5401cbabc01fa1b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03a4 expref: 805 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.794472:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc025633 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.794480:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44259 00010000:00010000:0.0:1783520935.794484:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc025633 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.794507:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35854 00010000:00010000:0.0:1783520935.794510:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520935.794516:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638585050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+805:2388:x1870153002388352:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.794524:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.794529:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c2c434c5/0xd5401cbabc01fa1b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5d9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03a4 expref: 805 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.794537:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9c000/2, rc 0, transno 4294975761 00010000:00010000:3.0:1783520935.794542:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c2c434c5/0xd5401cbabc01fa1b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5d9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03a4 expref: 805 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.794551:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993099008:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.794572:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514f99c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993100288:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.794591:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025641 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.794599:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+197:3627:x1870152993099392:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2213us (2374us total) trans 4294975762 rc 0/0 00000100:00100000:1.0:1783520935.794600:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1783520935.794608:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44262 00000100:00100000:1.0:1783520935.794613:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993101184:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.794628:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.794634:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc025633 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.794642:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.794647:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc025633 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.794648:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.794659:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc025633 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.794670:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc025633 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.794672:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c2c434c5/0xd5401cbabc01fa1b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5d9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03a4 expref: 805 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.794678:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc025633 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.794685:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.794688:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5d9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.794692:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02562c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.794694:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.794698:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.794699:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02562c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.794700:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.794709:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.794717:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000004:00080000:0.0:1783520935.794724:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9e680/2, rc 0, transno 4294975762 00000100:00100000:0.0:1783520935.794737:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993099392:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.794738:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002388480 00000100:00100000:2.0:1783520935.794746:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.794746:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002388480 00000100:00100000:3.0:1783520935.794747:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638585050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+804:2388:x1870153002388352:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 230us (416us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.794759:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35854 00010000:00010000:1.0:1783520935.794768:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c7b6824a/0xd5401cbabc01fa29 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03c0 expref: 805 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.794783:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1783520935.794794:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993101312:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.794803:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002388608 00000100:00100000:3.0:1783520935.794808:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35855 00000100:00100000:1.0:1783520935.794809:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002388608 00000100:00100000:3.0:1783520935.794817:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514f99c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+805:2388:x1870153002388480:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.794827:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.794827:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.794828:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.794835:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c7b6824a/0xd5401cbabc01fa29 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5db:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03c0 expref: 805 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.794845:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000228f4885/0xd5401cbabc01fa30 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03ce expref: 805 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.794858:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002388736 00000100:00100000:2.0:1783520935.794867:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002388736 00000100:00100000:1.0:1783520935.794871:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35856 00000100:00100000:0.0:1783520935.794871:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993101184 00000100:00100000:0.0:1783520935.794877:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993101184 00000100:00100000:1.0:1783520935.794886:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5ba300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+806:2387:x1870153002388608:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.794889:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c7b6824a/0xd5401cbabc01fa29 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5db:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03c0 expref: 806 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.794889:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a9f387e4/0xd5401cbabc01fa37 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03dc expref: 806 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.794900:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.794901:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993101312 00000100:00100000:0.0:1783520935.794905:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993101312 00010000:00010000:1.0:1783520935.794911:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000228f4885/0xd5401cbabc01fa30 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5dc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03ce expref: 806 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.794923:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44268 00000100:00100000:2.0:1783520935.794929:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35857 00000100:00100000:0.0:1783520935.794934:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635600850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+198:3625:x1870152993101184:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.794937:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c7b6824a/0xd5401cbabc01fa29 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5db:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03c0 expref: 806 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.794947:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5bbb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+806:2390:x1870153002388736:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.794947:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000228f4885/0xd5401cbabc01fa30 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5dc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03ce expref: 806 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.794948:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1504 count 1 00010000:00010000:3.0:1783520935.794950:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5db:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.794958:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5db:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.794962:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc02565d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5e0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.794963:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.794965:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025641 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.794973:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a9f387e4/0xd5401cbabc01fa37 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5dd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03dc expref: 805 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.794978:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025641 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.794978:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5dc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.794981:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000049393768/0xd5401cbabc01fa4c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0406 expref: 805 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.794989:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5dc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.795001:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e0:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.795002:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02563a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5dc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.795004:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a9f387e4/0xd5401cbabc01fa37 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5dd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03dc expref: 805 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.795007:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.795012:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.795017:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc02565d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.795030:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02563a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5dc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.795035:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514f99c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+805:2388:x1870153002388480:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 218us (327us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.795036:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5dd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.795044:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000049393768/0xd5401cbabc01fa4c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0406 expref: 805 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.795047:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5dd:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.795052:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35855 00010000:00010000:2.0:1783520935.795058:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025648 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5dd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.795058:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.795074:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000049393768/0xd5401cbabc01fa4c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0406 expref: 804 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.795076:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025648 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5dd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.795088:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5ba300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+804:2387:x1870153002388608:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 204us (343us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.795105:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993101440:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.795106:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.795106:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35856 00000100:00100000:2.0:1783520935.795134:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5bbb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+803:2390:x1870153002388736:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 192us (310us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.795134:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02562c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.795149:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35857 00000100:00100000:0.0:1783520935.795149:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470b40 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.795150:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02562c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.795162:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02562c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.795170:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02562c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.795170:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44269 00000100:00100000:0.0:1783520935.795181:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+198:3628:x1870152993101312:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.795184:0:8410:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000228f4885/0xd5401cbabc01fa30 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03ce expref: 802 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.795189:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00002000:00080000:0.0:1783520935.795195:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1505 count 1 00000100:00100000:3.0:1783520935.795196:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94361103b100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993100416:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.795207:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc025664 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5e1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.795211:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02563a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.795220:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000dd70101b/0xd5401cbabc01fa76 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0414 expref: 801 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.795221:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02563a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.795228:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e1:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.795232:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.795235:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02563a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.795236:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc025664 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.795243:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02563a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.795251:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02563a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.795258:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000dd70101b/0xd5401cbabc01fa76 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0414 expref: 801 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.795266:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:0.0:1783520935.795268:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000dd70101b/0xd5401cbabc01fa76 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0414 expref: 801 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.795285:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993101568:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.795287:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+199:3627:x1870152993099264:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2978us (3149us total) trans 4294975763 rc 0/0 00000100:00100000:1.0:1783520935.795297:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44261 00000100:00100000:0.0:1783520935.795315:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470180 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.795330:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520935.795335:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02562c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.795342:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.795346:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.795346:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02562c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.795353:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02563a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.795355:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+199:3626:x1870152993100032:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2274us (2348us total) trans 4294975764 rc 0/0 00010000:00010000:2.0:1783520935.795394:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.795398:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02563a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.795400:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9e300/2, rc 0, transno 4294975764 00010000:00010000:0.0:1783520935.795409:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02562c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.795412:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02563a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.795413:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993100032:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.795442:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02562c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.795443:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02563a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.795464:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc02562c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.795471:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44263 00010000:00010000:2.0:1783520935.795478:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02563a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.795486:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.795490:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.795493:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.795508:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.795511:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.795515:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.795527:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025641 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.795536:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025641 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.795545:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025641 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.795551:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993101696:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.795556:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025641 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.795558:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9ca80/2, rc 0, transno 4294975763 00000100:00100000:0.0:1783520935.795571:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993099264:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.795577:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.795578:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.795590:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c552d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993100928:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.795595:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.795599:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025641 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.795607:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.795611:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025641 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.795615:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993101696 00000100:00100000:1.0:1783520935.795621:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993101696 00010000:00010000:0.0:1783520935.795621:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025641 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.795637:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025641 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.795645:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025641 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.795653:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44270 00010000:00010000:0.0:1783520935.795665:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.795668:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+200:3626:x1870152993099904:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2521us (2583us total) trans 4294975765 rc 0/0 00000100:00100000:1.0:1783520935.795668:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+200:3628:x1870152993101696:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.795669:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.795673:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.795674:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02564f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.795685:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44264 00002000:00080000:1.0:1783520935.795695:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1506 count 1 00010000:00010000:1.0:1783520935.795709:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02566b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5e2:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.795716:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a9f387e4/0xd5401cbabc01fa37 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03dc expref: 801 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.795723:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008487807c/0xd5401cbabc01fa53 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0422 expref: 801 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.795724:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993101056:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.795734:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943611038a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993100672:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.795738:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e2:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.795740:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025656 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.795743:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.795748:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.795752:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025648 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.795759:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02566b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.795778:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.795781:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025648 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.795789:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025648 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.795793:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008487807c/0xd5401cbabc01fa53 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0422 expref: 800 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.795803:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002388864 00000100:00100000:0.0:1783520935.795807:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002388864 00010000:00010000:3.0:1783520935.795809:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025648 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.795812:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008487807c/0xd5401cbabc01fa53 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0422 expref: 800 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.795825:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ac01b8e6/0xd5401cbabc022b61 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3251 expref: 801 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.795828:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025648 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.795839:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993101952:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.795844:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.795853:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002388992 00000100:00100000:0.0:1783520935.795856:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002388992 00000100:00100000:2.0:1783520935.795870:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35858 00010000:00010000:0.0:1783520935.795872:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005167c805/0xd5401cbabc01fa45 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03f8 expref: 802 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.795885:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5bb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+802:2388:x1870153002388864:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.795885:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6300 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.795898:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.795902:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35859 00010000:00010000:2.0:1783520935.795908:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ac01b8e6/0xd5401cbabc022b61 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5de:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3251 expref: 802 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.795908:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993101824:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.795915:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5b8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+802:2388:x1870153002388992:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.795930:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.795933:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ac01b8e6/0xd5401cbabc022b61 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5de:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3251 expref: 802 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.795939:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005167c805/0xd5401cbabc01fa45 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5df:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03f8 expref: 802 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.795951:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.795959:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ac01b8e6/0xd5401cbabc022b61 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5de:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3251 expref: 802 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.795963:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005167c805/0xd5401cbabc01fa45 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5df:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03f8 expref: 802 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.795978:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+200:3628:x1870152993100160:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2487us (2663us total) trans 4294975766 rc 0/0 00000100:00100000:3.0:1783520935.795985:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44265 00010000:00010000:2.0:1783520935.795985:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5de:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.795989:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005167c805/0xd5401cbabc01fa45 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5df:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f03f8 expref: 801 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.795992:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5de:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.796001:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02564f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.796004:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993101824 00000004:00080000:1.0:1783520935.796004:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9c380/2, rc 0, transno 4294975766 00010000:00010000:0.0:1783520935.796006:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5df:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.796010:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993101824 00010000:00010000:0.0:1783520935.796015:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5df:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.796018:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993100160:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.796019:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02564f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.796021:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025656 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.796034:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44271 00000004:00080000:1.0:1783520935.796038:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9c700/2, rc 0, transno 4294975765 00010000:00010000:0.0:1783520935.796041:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025656 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.796046:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993099904:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.796047:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+200:3625:x1870152993101824:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.796053:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00002000:00080000:3.0:1783520935.796063:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1507 count 1 00010000:00010000:0.0:1783520935.796068:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.796080:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5bb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+800:2388:x1870153002388864:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 199us (341us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.796087:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35858 00000100:00100000:0.0:1783520935.796093:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5b8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+800:2388:x1870153002388992:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 180us (317us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.796102:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35859 00010000:00010000:3.0:1783520935.796123:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025672 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5e3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.796123:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.796128:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025648 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.796134:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.796135:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000df839f65/0xd5401cbabc01fa5a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0430 expref: 798 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.796138:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025648 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.796143:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e3:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.796146:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.796146:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025648 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.796150:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025672 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.796158:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025648 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.796165:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993101440:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.796167:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025648 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.796173:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000df839f65/0xd5401cbabc01fa5a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0430 expref: 798 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.796184:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000df839f65/0xd5401cbabc01fa5a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0430 expref: 798 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.796187:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc02565d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.796189:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.796193:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.796196:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.796199:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c552d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993102080:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.796230:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.796239:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993102208:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.796262:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993102336:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.796269:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.796279:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002389120 00000100:00100000:2.0:1783520935.796283:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002389120 00010000:00010000:2.0:1783520935.796299:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000049393768/0xd5401cbabc01fa4c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0406 expref: 799 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.796304:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993102208 00000100:00100000:0.0:1783520935.796305:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.796309:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993102208 00000100:00100000:2.0:1783520935.796330:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35860 00000100:00100000:2.0:1783520935.796339:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5baa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+799:2388:x1870153002389120:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.796339:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993102336 00000100:00100000:1.0:1783520935.796343:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993102336 00010000:00010000:2.0:1783520935.796348:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.796353:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000049393768/0xd5401cbabc01fa4c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0406 expref: 799 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.796380:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993101568:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.796392:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000049393768/0xd5401cbabc01fa4c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5e0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0406 expref: 799 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.796396:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc025664 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.796400:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44272 00010000:00010000:2.0:1783520935.796416:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000049393768/0xd5401cbabc01fa4c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5e0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0406 expref: 799 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.796417:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+202:3626:x1870152993102208:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.796429:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435129679c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.796436:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e0:0x0].0x0 since=0s v=(0 0) 00002000:00080000:1.0:1783520935.796439:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1508 count 1 00010000:00010000:2.0:1783520935.796441:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.796447:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc02565d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.796448:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44273 00010000:00010000:1.0:1783520935.796454:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025679 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5e4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.796460:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc02565d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.796463:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435079f1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+202:3627:x1870152993102336:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.796465:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02564f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.796484:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1509 count 1 00010000:00010000:1.0:1783520935.796485:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006f6d7dfc/0xd5401cbabc01fa61 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f043e expref: 798 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.796493:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02564f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.796494:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.796494:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e4:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.796512:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025680 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5e5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.796514:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.796516:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02564f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.796522:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025679 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.796534:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5baa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+798:2388:x1870153002389120:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 193us (302us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.796543:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000da49b4c7/0xd5401cbabc01fa68 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f044c expref: 798 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.796544:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02564f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.796558:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006f6d7dfc/0xd5401cbabc01fa61 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f043e expref: 798 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.796565:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35860 00010000:00010000:0.0:1783520935.796570:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e5:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.796579:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.796584:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006f6d7dfc/0xd5401cbabc01fa61 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f043e expref: 797 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.796588:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025680 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.796605:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.796610:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b1500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993102464:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.796613:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:2.0:1783520935.796619:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.796621:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02564f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.796630:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.796635:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02564f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.796646:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6cc0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.796650:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02564f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.796668:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc02565d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.796668:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02564f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.796688:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc02565d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.796690:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02564f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.796692:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+203:3628:x1870152993100544:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2802us (3017us total) trans 4294975767 rc 0/0 00000100:00100000:2.0:1783520935.796708:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002389248 00000100:00100000:3.0:1783520935.796712:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44266 00000100:00100000:2.0:1783520935.796714:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002389248 00010000:00010000:1.0:1783520935.796716:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc02565d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.796720:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.796723:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.796725:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.796729:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc02565d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.796734:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000dd70101b/0xd5401cbabc01fa76 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0414 expref: 798 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.796753:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000da49b4c7/0xd5401cbabc01fa68 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f044c expref: 798 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.796758:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.796762:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000da49b4c7/0xd5401cbabc01fa68 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f044c expref: 798 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.796770:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993102592:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.796771:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.796777:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc02565d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.796786:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfa400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.796791:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc02565d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.796794:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470180 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.796803:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc02565d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.796810:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025656 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.796816:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc02565d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.796822:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025656 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.796831:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000855d7374/0xd5401cbabc02565d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.796837:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635600850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+203:3625:x1870152993101184:12345-0@lo:6:osp-syn-1-0.0 Request processed in 1902us (2189us total) trans 4294975768 rc 0/0 00000100:00100000:1.0:1783520935.796856:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44268 00010000:00010000:0.0:1783520935.796857:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025656 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.796866:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025656 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.796868:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.796872:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.796878:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000004:00080000:3.0:1783520935.796880:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ad1f80/2, rc 0, transno 4294975767 00010000:00010000:0.0:1783520935.796882:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to regular blp list, 0 blwis in pool 00010000:00010000:2.0:1783520935.796884:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025656 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.796892:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.796893:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb600 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:3.0:1783520935.796894:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993100544:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.796899:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025656 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.796908:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025656 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.796910:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e1c5880/2, rc 0, transno 4294975768 00000100:00100000:0.0:1783520935.796920:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993101184:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.796924:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025656 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.796932:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f07fd018/0xd5401cbabc025656 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.796948:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:2.0:1783520935.796950:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.796955:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.796958:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.796959:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993102720:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.796976:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35861 00000100:00100000:1.0:1783520935.796989:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1783520935.796990:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638584450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+798:2388:x1870153002389248:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.796993:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.797001:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993102848:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.797004:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.797012:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000dd70101b/0xd5401cbabc01fa76 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0414 expref: 798 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.797020:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000dd70101b/0xd5401cbabc01fa76 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5e1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0414 expref: 798 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.797031:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000dd70101b/0xd5401cbabc01fa76 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5e1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0414 expref: 798 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.797038:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.797046:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.797050:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.797053:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc025664 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.797059:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc025664 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.797075:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.797082:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993101952:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.797096:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638584450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+797:2388:x1870153002389248:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 111us (482us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.797103:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+203:3626:x1870152993100800:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3058us (3143us total) trans 4294975769 rc 0/0 00010000:00010000:1.0:1783520935.797104:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02566b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.797107:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35861 00000100:00100000:0.0:1783520935.797120:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44267 00000004:00080000:1.0:1783520935.797148:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9fb80/2, rc 0, transno 4294975769 00000100:00100000:1.0:1783520935.797162:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993100800:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.797182:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.797206:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002389376 00000100:00100000:2.0:1783520935.797210:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002389376 00000100:00100000:1.0:1783520935.797213:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993102976:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.797229:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008487807c/0xd5401cbabc01fa53 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0422 expref: 797 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.797243:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993102720 00000100:00100000:1.0:1783520935.797247:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.797249:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993102720 00000100:00100000:2.0:1783520935.797258:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35862 00000100:00100000:2.0:1783520935.797270:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5ba300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+797:2388:x1870153002389376:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.797279:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993102848 00010000:00010000:2.0:1783520935.797280:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.797285:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993102848 00000100:00100000:1.0:1783520935.797286:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993102976 00010000:00010000:2.0:1783520935.797289:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008487807c/0xd5401cbabc01fa53 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0422 expref: 797 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.797291:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993102976 00010000:00010000:2.0:1783520935.797302:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008487807c/0xd5401cbabc01fa53 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5e2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0422 expref: 797 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.797306:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44274 00000100:00100000:0.0:1783520935.797317:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44275 00000100:00100000:3.0:1783520935.797318:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+205:3627:x1870152993102720:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.797320:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008487807c/0xd5401cbabc01fa53 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5e2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0422 expref: 797 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.797328:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+205:3626:x1870152993102848:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520935.797332:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1510 count 1 00010000:00010000:2.0:1783520935.797333:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.797337:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e2:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.797338:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44276 00002000:00080000:0.0:1783520935.797340:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1511 count 1 00010000:00010000:3.0:1783520935.797343:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025687 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5e6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.797344:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02566b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.797350:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+205:3626:x1870152993102976:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.797353:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02568e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5e7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.797416:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc022a50 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3140 expref: 796 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.797427:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x5e6:0x0].0x0 since=36s v=(0 0) 00010000:00010000:3.0:1783520935.797432:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.797433:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02566b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.797439:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025687 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.797455:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1512 count 1 00010000:00010000:0.0:1783520935.797458:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000539aa150/0xd5401cbabc01fa7d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0468 expref: 796 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.797466:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e7:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.797469:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.797473:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025695 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5e8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.797473:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02568e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.797482:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.797513:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000009aab39c/0xd5401cbabc01fa8b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0476 expref: 796 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.797522:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5ba300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+796:2388:x1870153002389376:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 254us (339us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.797524:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e8:0x0].0x0 since=37s v=(0 0) 00000100:00100000:2.0:1783520935.797530:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35862 00010000:00010000:1.0:1783520935.797530:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.797534:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc025664 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.797538:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025695 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.797544:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc025664 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.797558:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc025664 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.797565:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc025664 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.797571:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000009aab39c/0xd5401cbabc01fa8b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0476 expref: 795 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.797578:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.797593:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000009aab39c/0xd5401cbabc01fa8b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0476 expref: 795 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.797605:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b1880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993103104:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.797656:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6300 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.797658:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+206:3628:x1870152993101312:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2478us (2831us total) trans 4294975770 rc 0/0 00000100:00100000:0.0:1783520935.797666:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44269 00010000:00010000:1.0:1783520935.797676:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02566b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.797686:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02566b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.797688:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.797695:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc025664 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.797699:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02566b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.797708:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfbe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.797714:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc025664 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.797716:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02566b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.797734:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc025664 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.797742:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.797748:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc025664 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.797756:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007b43d4cc/0xd5401cbabc025664 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.797771:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.797775:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02566b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.797782:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.797786:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02566b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.797794:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02566b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.797811:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+206:3628:x1870152993101696:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2149us (2232us total) trans 4294975771 rc 0/0 00010000:00010000:0.0:1783520935.797812:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02566b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.797825:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44270 00010000:00010000:0.0:1783520935.797827:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000dc1a697e/0xd5401cbabc02566b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.797843:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.797845:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.797848:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.797855:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520935.797873:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435079f3b80/2, rc 0, transno 4294975771 00000100:00100000:0.0:1783520935.797885:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993101696:0@lo:6:osp-syn-1-0.0 00000004:00080000:0.0:1783520935.797898:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ad1180/2, rc 0, transno 4294975770 00000100:00100000:0.0:1783520935.797904:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993101312:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.797931:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993103232:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.797957:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.797967:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.797971:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b1500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993102464:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.797991:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025679 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.797997:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1783520935.798007:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993103360:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.798023:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993103232 00000100:00100000:1.0:1783520935.798029:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993103232 00000100:00100000:2.0:1783520935.798033:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.798038:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.798064:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993103360 00000100:00100000:2.0:1783520935.798068:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.798070:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993103360 00000100:00100000:0.0:1783520935.798079:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44277 00000100:00100000:2.0:1783520935.798090:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002389632 00000100:00100000:0.0:1783520935.798090:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+207:3627:x1870152993103232:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.798096:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002389632 00000100:00100000:1.0:1783520935.798103:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44278 00002000:00080000:0.0:1783520935.798107:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1513 count 1 00010000:00010000:2.0:1783520935.798115:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006f6d7dfc/0xd5401cbabc01fa61 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f043e expref: 796 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.798116:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+207:3627:x1870152993103360:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.798121:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02569c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5e9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.798137:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1514 count 1 00000100:00100000:2.0:1783520935.798141:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002389504 00000100:00100000:2.0:1783520935.798145:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002389504 00010000:00010000:0.0:1783520935.798150:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000603ffa40/0xd5401cbabc01fa84 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0484 expref: 797 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.798153:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0256a3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5ea:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.798156:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000df839f65/0xd5401cbabc01fa5a lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0430 expref: 797 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.798167:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e9:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.798176:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.798182:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02569c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.798184:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000015a234e/0xd5401cbabc01fa92 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0492 expref: 797 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.798188:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002389760 00000100:00100000:2.0:1783520935.798192:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002389760 00010000:00010000:1.0:1783520935.798202:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ea:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.798206:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000da49b4c7/0xd5401cbabc01fa68 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f044c expref: 798 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.798208:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ea:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.798213:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0256a3 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.798216:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000603ffa40/0xd5401cbabc01fa84 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0484 expref: 798 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.798244:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000603ffa40/0xd5401cbabc01fa84 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0484 expref: 798 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.798257:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502250380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993103488:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.798258:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000015a234e/0xd5401cbabc01fa92 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0492 expref: 798 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.798262:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35863 00010000:00010000:1.0:1783520935.798274:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000015a234e/0xd5401cbabc01fa92 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0492 expref: 798 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.798279:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5bbb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+798:2388:x1870153002389632:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.798290:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b1500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993103616:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.798293:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.798296:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470780 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.798302:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006f6d7dfc/0xd5401cbabc01fa61 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f043e expref: 798 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.798316:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006f6d7dfc/0xd5401cbabc01fa61 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5e4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f043e expref: 798 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.798316:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6cc0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.798323:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993102592:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.798341:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f6d7dfc/0xd5401cbabc01fa61 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5e4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f043e expref: 798 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.798342:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025680 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.798393:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.798399:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.798404:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025679 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.798413:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025679 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.798420:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000539aa150/0xd5401cbabc01fa7d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0468 expref: 797 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.798430:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000539aa150/0xd5401cbabc01fa7d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0468 expref: 797 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.798439:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.798439:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b3800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993103744:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.798447:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025679 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.798457:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025679 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.798467:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025679 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.798468:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5bbb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+797:2388:x1870153002389632:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 193us (511us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.798481:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6840 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.798484:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35863 00010000:00010000:0.0:1783520935.798484:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025679 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.798515:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520935.798521:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35864 00010000:00010000:1.0:1783520935.798527:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:2.0:1783520935.798531:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5bb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+796:2388:x1870153002389504:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.798534:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025679 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.798539:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.798541:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.798546:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000df839f65/0xd5401cbabc01fa5a lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5e3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0430 expref: 796 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.798546:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025679 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.798567:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025679 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.798568:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000df839f65/0xd5401cbabc01fa5a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0430 expref: 796 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.798590:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025679 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.798595:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e3:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.798597:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+208:3626:x1870152993102208:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2184us (2327us total) trans 4294975772 rc 0/0 00010000:00010000:2.0:1783520935.798605:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.798607:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc025679 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.798612:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44272 00010000:00010000:2.0:1783520935.798614:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025672 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.798636:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025672 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5e3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.798654:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.798657:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.798659:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.798661:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.798684:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5bb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+796:2388:x1870153002389504:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 153us (650us total) trans 0 rc 0/0 00000004:00080000:1.0:1783520935.798686:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435079f0e00/2, rc 0, transno 4294975772 00000100:00100000:2.0:1783520935.798695:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35864 00000100:00100000:1.0:1783520935.798700:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993102208:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.798719:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35865 00000100:00100000:2.0:1783520935.798730:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5b8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+795:2389:x1870153002389760:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.798738:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.798744:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000da49b4c7/0xd5401cbabc01fa68 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f044c expref: 795 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.798747:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:2.0:1783520935.798760:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000da49b4c7/0xd5401cbabc01fa68 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5e5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f044c expref: 795 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.798761:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502253800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993103872:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.798781:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000da49b4c7/0xd5401cbabc01fa68 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5e5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f044c expref: 795 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.798794:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.798795:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.798799:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.798804:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025680 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.798812:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025680 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.798816:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993103872 00000100:00100000:0.0:1783520935.798822:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993103872 00010000:00010000:2.0:1783520935.798843:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.798851:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44279 00010000:00010000:1.0:1783520935.798858:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025680 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.798860:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+208:3626:x1870152993103872:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.798866:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5b8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+794:2389:x1870153002389760:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 136us (798us total) trans 0 rc 0/0 00002000:00080000:0.0:1783520935.798878:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1515 count 1 00010000:00010000:1.0:1783520935.798880:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025680 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.798885:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35865 00010000:00010000:1.0:1783520935.798893:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025680 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.798895:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0256aa lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5eb:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.798901:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025680 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.798917:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006533e270/0xd5401cbabc01fa99 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f04a0 expref: 793 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.798925:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.798926:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5eb:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.798931:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5eb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.798932:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.798937:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0256aa lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.798939:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025680 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.798962:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.798968:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025680 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.798976:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025680 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.798979:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006533e270/0xd5401cbabc01fa99 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04a0 expref: 793 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.798990:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025680 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.798995:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006533e270/0xd5401cbabc01fa99 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04a0 expref: 793 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.799002:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a5d93cec/0xd5401cbabc025680 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.799016:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435079f1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+209:3627:x1870152993102336:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2556us (2714us total) trans 4294975773 rc 0/0 00000100:00100000:0.0:1783520935.799019:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502252a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993104000:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.799026:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44273 00010000:00010000:2.0:1783520935.799029:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.799032:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.799035:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.799051:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470900 going to sleep for 11 seconds 00000004:00080000:0.0:1783520935.799081:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435079f3800/2, rc 0, transno 4294975773 00000100:00100000:1.0:1783520935.799086:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b1880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993103104:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.799093:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993102336:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.799103:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025695 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.799143:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502252680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993104128:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.799154:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.799177:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.799203:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993104128 00000100:00100000:0.0:1783520935.799207:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993104128 00000100:00100000:0.0:1783520935.799240:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44280 00000100:00100000:0.0:1783520935.799249:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+209:3626:x1870152993104128:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.799260:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1516 count 1 00010000:00010000:0.0:1783520935.799271:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0256b1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5ec:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.799282:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006164c652/0xd5401cbabc01faa0 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f04ae expref: 793 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.799290:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ec:0x0].0x0 since=37s v=(0 0) 00000100:00100000:2.0:1783520935.799292:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002389888 00010000:00010000:0.0:1783520935.799294:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ec:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.799299:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002389888 00010000:00010000:0.0:1783520935.799300:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0256b1 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.799323:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000009aab39c/0xd5401cbabc01fa8b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0476 expref: 794 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.799325:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006164c652/0xd5401cbabc01faa0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04ae expref: 794 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.799348:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006164c652/0xd5401cbabc01faa0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04ae expref: 794 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.799378:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502253100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993104256:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.799385:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35866 00000100:00100000:2.0:1783520935.799395:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5baa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+794:2388:x1870153002389888:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.799405:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.799410:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000009aab39c/0xd5401cbabc01fa8b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0476 expref: 794 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.799424:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000009aab39c/0xd5401cbabc01fa8b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5e8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0476 expref: 794 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.799426:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470180 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.799440:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000009aab39c/0xd5401cbabc01fa8b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5e8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0476 expref: 794 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.799447:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.799449:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.799452:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025695 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.799457:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025695 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.799471:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.799484:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025695 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.799493:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5baa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+793:2388:x1870153002389888:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 99us (339us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.799508:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025695 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.799517:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35866 00010000:00010000:1.0:1783520935.799518:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025695 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.799525:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025695 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.799537:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.799549:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.799555:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025695 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.799564:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.799568:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025695 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.799576:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025695 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.799589:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025695 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.799598:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001e04e98c/0xd5401cbabc025695 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.799608:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+210:3626:x1870152993102976:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2258us (2359us total) trans 4294975774 rc 0/0 00000100:00100000:1.0:1783520935.799621:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44276 00010000:00010000:0.0:1783520935.799627:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.799631:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.799633:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520935.799656:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b0000/2, rc 0, transno 4294975774 00000100:00100000:1.0:1783520935.799668:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993102976:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.799707:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1783520935.799716:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993104384:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.799743:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.799761:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993104384 00000100:00100000:1.0:1783520935.799766:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993104384 00000100:00100000:1.0:1783520935.799790:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44281 00000100:00100000:1.0:1783520935.799800:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517244e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+210:3628:x1870152993104384:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.799811:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1517 count 1 00010000:00010000:1.0:1783520935.799820:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc0256b8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5ed:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.799829:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000267f983d/0xd5401cbabc01faa7 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f04bc expref: 792 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.799833:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ed:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.799836:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ed:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.799839:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc0256b8 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.799854:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000267f983d/0xd5401cbabc01faa7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04bc expref: 792 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.799861:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000267f983d/0xd5401cbabc01faa7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04bc expref: 792 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.799868:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993104512:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.799901:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c369480 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.800285:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502250380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993103488:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.800302:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02569c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.800398:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b1500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993103616:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.800412:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b3800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993103744:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.800416:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0256a3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.800432:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02568e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.800484:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.800484:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502252a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993104000:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.800516:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0256aa lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.800520:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002390144 00000100:00100000:0.0:1783520935.800527:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002390144 00000100:00100000:2.0:1783520935.800528:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.800548:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000015a234e/0xd5401cbabc01fa92 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0492 expref: 793 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.800549:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002390400 00000100:00100000:2.0:1783520935.800553:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002390400 00010000:00010000:2.0:1783520935.800569:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006533e270/0xd5401cbabc01fa99 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04a0 expref: 794 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.800580:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35867 00000100:00100000:0.0:1783520935.800590:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5b8380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+794:2387:x1870153002390144:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.800598:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.800603:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000015a234e/0xd5401cbabc01fa92 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ea:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0492 expref: 794 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.800615:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35868 00010000:00010000:0.0:1783520935.800625:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000015a234e/0xd5401cbabc01fa92 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5ea:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0492 expref: 794 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.800634:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638586450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+794:2387:x1870153002390400:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.800645:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000015a234e/0xd5401cbabc01fa92 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5ea:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0492 expref: 794 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.800646:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.800656:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006533e270/0xd5401cbabc01fa99 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5eb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04a0 expref: 794 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.800661:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ea:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.800669:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ea:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.800675:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006533e270/0xd5401cbabc01fa99 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5eb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04a0 expref: 793 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.800675:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0256a3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.800685:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0256a3 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.800706:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006533e270/0xd5401cbabc01fa99 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5eb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04a0 expref: 793 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.800716:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.800719:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5eb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.800724:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5eb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.800728:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0256aa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.800742:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0256aa lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.800746:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0256a3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.800747:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5b8380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+792:2387:x1870153002390144:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 156us (262us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.800765:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35867 00010000:00010000:1.0:1783520935.800767:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0256a3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.800774:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.800778:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0256a3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.800786:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0256a3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.800800:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520935.800801:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638586450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+791:2387:x1870153002390400:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 171us (274us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.800811:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35868 00010000:00010000:0.0:1783520935.800811:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.800818:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0256a3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.800829:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.800833:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0256a3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.800847:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0256a3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.800861:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0256a3 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.800870:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008faaf9b7/0xd5401cbabc0256a3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.800891:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.800892:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+211:3627:x1870152993103360:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2780us (2853us total) trans 4294975775 rc 0/0 00010000:00010000:0.0:1783520935.800894:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.800898:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.800902:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44278 00010000:00010000:0.0:1783520935.800904:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.800946:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000004:00080000:1.0:1783520935.800948:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b0e00/2, rc 0, transno 4294975775 00000100:00100000:1.0:1783520935.800964:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993103360:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.800988:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0256aa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.801003:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0256aa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.801008:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002390016 00010000:00010000:1.0:1783520935.801012:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0256aa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.801014:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002390016 00000100:00100000:0.0:1783520935.801014:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502252a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993104640:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.801028:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0256aa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.801039:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000603ffa40/0xd5401cbabc01fa84 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0484 expref: 791 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.801045:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.801055:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.801057:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.801063:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0256aa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.801065:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002390272 00000100:00100000:2.0:1783520935.801070:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002390272 00010000:00010000:1.0:1783520935.801072:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfa200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.801077:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0256aa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.801083:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0256aa lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.801092:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000539aa150/0xd5401cbabc01fa7d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0468 expref: 792 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.801092:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35869 00010000:00010000:1.0:1783520935.801095:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0256aa lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.801105:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5ba300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+792:2388:x1870153002390016:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.801116:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c3fe08ef/0xd5401cbabc0256aa lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.801119:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.801127:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000603ffa40/0xd5401cbabc01fa84 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0484 expref: 792 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.801141:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.801141:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000603ffa40/0xd5401cbabc01fa84 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5e9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0484 expref: 792 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.801142:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35870 00010000:00010000:1.0:1783520935.801145:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.801149:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.801158:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5bbb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+792:2388:x1870153002390272:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.801160:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000603ffa40/0xd5401cbabc01fa84 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5e9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0484 expref: 792 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.801167:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993104640 00010000:00010000:2.0:1783520935.801170:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.801171:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e9:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.801172:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993104640 00010000:00010000:0.0:1783520935.801177:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.801180:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000539aa150/0xd5401cbabc01fa7d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5e7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0468 expref: 791 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.801181:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02569c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.801195:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44282 00010000:00010000:0.0:1783520935.801198:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02569c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.801205:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502250380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+211:3628:x1870152993104640:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.801223:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1518 count 1 00010000:00010000:2.0:1783520935.801226:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000539aa150/0xd5401cbabc01fa7d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5e7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0468 expref: 791 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.801226:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.801235:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0256bf lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5ee:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.801247:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000539aa150/0xd5401cbabc01fa7d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5e7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0468 expref: 791 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.801248:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008508a85d/0xd5401cbabc02293f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f302f expref: 791 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.801248:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5ba300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+791:2388:x1870153002390016:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 145us (359us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.801265:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35869 00010000:00010000:1.0:1783520935.801270:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x5ee:0x0].0x0 since=36s v=(0 0) 00010000:00010000:1.0:1783520935.801274:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ee:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.801277:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0256bf lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.801278:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.801287:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.801289:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02569c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.801294:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02568e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.801297:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02569c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.801310:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008508a85d/0xd5401cbabc02293f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f302f expref: 789 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.801313:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02568e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.801316:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02569c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.801338:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02569c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.801340:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008508a85d/0xd5401cbabc02293f lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f302f expref: 789 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.801382:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.801392:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b1500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993104768:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.801394:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.801399:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02569c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.801399:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to regular blp list, 0 blwis in pool 00010000:00010000:2.0:1783520935.801407:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.801410:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02569c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.801416:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02569c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.801429:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02569c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.801433:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c67c07c2/0xd5401cbabc02569c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.801451:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.801451:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6cc0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.801454:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.801459:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.801506:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+212:3627:x1870152993103232:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3407us (3527us total) trans 4294975776 rc 0/0 00000100:00100000:0.0:1783520935.801515:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44277 00000004:00080000:1.0:1783520935.801525:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b0a80/2, rc 0, transno 4294975776 00000100:00100000:1.0:1783520935.801540:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993103232:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.801586:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515346d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993104896:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.801619:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.801654:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+212:3626:x1870152993103872:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2793us (2858us total) trans 4294975777 rc 0/0 00000100:00100000:2.0:1783520935.801664:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.801668:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44279 00000100:00100000:0.0:1783520935.801670:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502253100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993104256:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.801686:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0256b1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.801689:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993104896 00000100:00100000:1.0:1783520935.801696:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993104896 00010000:00010000:0.0:1783520935.801708:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02568e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.801720:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.801720:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02568e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.801730:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44283 00010000:00010000:0.0:1783520935.801735:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02568e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.801743:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515345500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+212:3625:x1870152993104896:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.801751:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002390528 00010000:00010000:0.0:1783520935.801751:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02568e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.801758:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002390528 00002000:00080000:1.0:1783520935.801761:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1519 count 1 00010000:00010000:0.0:1783520935.801767:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.801775:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0256c6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5ef:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.801776:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.801781:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006164c652/0xd5401cbabc01faa0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04ae expref: 790 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.801782:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02568e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.801799:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d208c8b1/0xd5401cbabc01fab5 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f04d8 expref: 790 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.801821:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002390656 00010000:00010000:1.0:1783520935.801821:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ef:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.801823:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb200 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520935.801825:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002390656 00010000:00010000:1.0:1783520935.801830:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ef:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.801830:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02568e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.801838:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0256c6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.801844:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02568e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.801845:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000267f983d/0xd5401cbabc01faa7 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04bc expref: 791 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.801870:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02568e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.801877:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000692b0dc3/0xd5401cbabc02568e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.801878:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d208c8b1/0xd5401cbabc01fab5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04d8 expref: 791 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.801885:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35871 00010000:00010000:0.0:1783520935.801900:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.801902:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5bb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+791:2387:x1870153002390528:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.801902:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d208c8b1/0xd5401cbabc01fab5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04d8 expref: 791 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.801903:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.801906:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.801919:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.801920:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35872 00000100:00100000:1.0:1783520935.801924:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b2680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993105024:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.801929:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006164c652/0xd5401cbabc01faa0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ec:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04ae expref: 791 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.801931:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5b8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+791:2387:x1870153002390656:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.801943:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.801952:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006164c652/0xd5401cbabc01faa0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5ec:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04ae expref: 791 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.801952:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000267f983d/0xd5401cbabc01faa7 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5ed:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04bc expref: 791 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.801963:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6300 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.801976:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000267f983d/0xd5401cbabc01faa7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ed:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04bc expref: 791 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.801983:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006164c652/0xd5401cbabc01faa0 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5ec:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04ae expref: 791 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.801998:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502253800/2, rc 0, transno 4294975777 00010000:00010000:0.0:1783520935.802009:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ed:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.802010:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ec:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.802015:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ec:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.802017:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ed:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.802018:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502253800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993103872:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.802021:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0256b1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.802027:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc0256b8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ed:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.802043:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0256b1 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.802044:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc0256b8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5ed:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.802061:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000267f983d/0xd5401cbabc01faa7 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5ed:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04bc expref: 790 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.802074:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993104512:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.802077:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.802093:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc0256b8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.802093:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1783520935.802102:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502253100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993105152:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.802104:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5bb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+789:2387:x1870153002390528:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 205us (444us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.802116:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35871 00000100:00100000:0.0:1783520935.802132:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.802160:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993105152 00000100:00100000:1.0:1783520935.802166:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993105152 00010000:00010000:0.0:1783520935.802168:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.802190:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5b8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+788:2387:x1870153002390656:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 262us (473us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.802196:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35872 00000100:00100000:1.0:1783520935.802204:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44284 00000100:00100000:1.0:1783520935.802216:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350794f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+213:3625:x1870152993105152:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.802221:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0256b1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.802227:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0256b1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.802230:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1520 count 1 00010000:00010000:0.0:1783520935.802235:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0256b1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.802241:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0256b1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.802246:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0256cd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5f0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.802253:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.802260:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f2004678/0xd5401cbabc01fabc lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f04e6 expref: 787 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.802261:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.802266:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0256b1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.802277:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f0:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.802279:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfbc00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.802285:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0256b1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.802286:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f0:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.802295:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5bbb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+787:2388:x1870153002390272:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 1138us (1345us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.802297:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0256cd lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.802298:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0256b1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.802313:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35870 00010000:00010000:0.0:1783520935.802322:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0256b1 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.802328:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000001e3cafb/0xd5401cbabc0256b1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.802340:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f2004678/0xd5401cbabc01fabc lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04e6 expref: 786 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.802344:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.802348:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.802353:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.802382:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f2004678/0xd5401cbabc01fabc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04e6 expref: 786 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.802396:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993105280:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.802453:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6840 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.802467:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+214:3626:x1870152993104128:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3219us (3290us total) trans 4294975778 rc 0/0 00000100:00100000:0.0:1783520935.802476:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44280 00000004:00080000:1.0:1783520935.802482:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502252680/2, rc 0, transno 4294975778 00000100:00100000:1.0:1783520935.802512:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502252680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993104128:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.802540:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b1500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993104768:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.802556:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0256bf lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.802561:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350794e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993105408:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.802581:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc0256b8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.802590:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc0256b8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.802601:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.802604:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc0256b8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.802613:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc0256b8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.802629:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.802636:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+214:3626:x1870152993102848:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5310us (5601us total) trans 4294975779 rc 0/0 00010000:00010000:1.0:1783520935.802638:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.802646:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc0256b8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.802649:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44275 00010000:00010000:1.0:1783520935.802658:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.802663:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc0256b8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.802671:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993105408 00010000:00010000:1.0:1783520935.802672:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc0256b8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.802677:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993105408 00000100:00100000:2.0:1783520935.802688:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.802689:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc0256b8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.802698:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000007d3e164/0xd5401cbabc0256b8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.802711:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44285 00000100:00100000:2.0:1783520935.802717:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002390784 00010000:00010000:1.0:1783520935.802719:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.802723:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002390784 00000100:00100000:0.0:1783520935.802723:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5baa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+214:3627:x1870152993105408:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.802725:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.802729:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00002000:00080000:0.0:1783520935.802740:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1521 count 1 00010000:00010000:2.0:1783520935.802748:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008508a85d/0xd5401cbabc02293f lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f302f expref: 787 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.802751:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b2a00/2, rc 0, transno 4294975779 00010000:00010000:0.0:1783520935.802754:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0256d4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5f1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.802766:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993102848:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.802777:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c4d4ca72/0xd5401cbabc01fac3 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f04f4 expref: 787 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.802786:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f1:0x0].0x0 since=37s v=(0 0) 00000100:00100000:2.0:1783520935.802787:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35873 00010000:00010000:0.0:1783520935.802791:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.802797:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0256d4 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.802801:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5b8380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+787:2389:x1870153002390784:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.802816:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.802823:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008508a85d/0xd5401cbabc02293f lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5ee:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f302f expref: 787 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.802828:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993105536:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.802840:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008508a85d/0xd5401cbabc02293f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5ee:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f302f expref: 787 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.802855:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008508a85d/0xd5401cbabc02293f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5ee:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f302f expref: 787 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.802864:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.802875:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ee:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.802878:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ee:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.802883:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0256bf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.802892:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0256bf lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.802898:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993105536 00000100:00100000:0.0:1783520935.802902:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993105536 00010000:00010000:2.0:1783520935.802914:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.802917:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517244e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+216:3628:x1870152993104384:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3116us (3170us total) trans 4294975780 rc 0/0 00000100:00100000:0.0:1783520935.802926:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44286 00000100:00100000:0.0:1783520935.802932:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350794df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+216:3628:x1870152993105536:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.802937:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44281 00002000:00080000:0.0:1783520935.802944:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1522 count 1 00000100:00100000:2.0:1783520935.802947:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5b8380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+786:2389:x1870153002390784:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 149us (260us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.802956:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35873 00010000:00010000:0.0:1783520935.802956:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0256db lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5f2:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.802969:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0256bf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.802974:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004c350f36/0xd5401cbabc01faca lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0502 expref: 785 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.802989:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0256bf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.802991:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f2:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.803000:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.803005:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0256bf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.803006:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0256db lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.803027:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0256bf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.803046:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004c350f36/0xd5401cbabc01faca lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0502 expref: 785 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.803049:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.803054:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.803062:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0256bf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.803064:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004c350f36/0xd5401cbabc01faca lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0502 expref: 785 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.803082:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.803088:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0256bf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.803089:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9df80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993105664:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.803103:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0256bf lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.803116:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0256bf lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.803124:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000018903c53/0xd5401cbabc0256bf lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.803128:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470180 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.803137:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502250380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+216:3628:x1870152993104640:12345-0@lo:6:osp-syn-1-0.0 Request processed in 1931us (2081us total) trans 4294975781 rc 0/0 00000100:00100000:1.0:1783520935.803145:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44282 00010000:00010000:2.0:1783520935.803148:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.803153:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.803155:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520935.803160:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502252a00/2, rc 0, transno 4294975781 00000100:00100000:0.0:1783520935.803170:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502252a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993104640:0@lo:6:osp-syn-1-0.0 00000004:00080000:0.0:1783520935.803180:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435079f0700/2, rc 0, transno 4294975780 00000100:00100000:0.0:1783520935.803182:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993104384:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.803229:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993105920:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.803243:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c4d4ca72/0xd5401cbabc01fac3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04f4 expref: 785 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.803260:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c4d4ca72/0xd5401cbabc01fac3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04f4 expref: 785 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.803269:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.803280:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993106048:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.803298:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993105920 00000100:00100000:1.0:1783520935.803299:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1783520935.803305:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993105920 00000100:00100000:1.0:1783520935.803310:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993105792:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.803315:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124706c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.803340:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44287 00000100:00100000:1.0:1783520935.803343:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.803353:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+216:3626:x1870152993105920:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.803390:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993105792 00002000:00080000:2.0:1783520935.803391:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1524 count 1 00000100:00100000:0.0:1783520935.803397:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993105792 00010000:00010000:2.0:1783520935.803405:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0256e2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5f4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.803417:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007872e47b/0xd5401cbabc01fad8 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f051e expref: 785 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.803424:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f4:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.803428:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f4:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.803429:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44288 00010000:00010000:2.0:1783520935.803434:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0256e2 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.803442:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+217:3626:x1870152993105792:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.803457:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1523 count 1 00000100:00100000:1.0:1783520935.803460:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993105280:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.803461:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007872e47b/0xd5401cbabc01fad8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f051e expref: 785 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.803468:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0256e9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5f3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.803487:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007872e47b/0xd5401cbabc01fad8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f051e expref: 785 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.803487:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0256cd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.803487:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002cd7b47c/0xd5401cbabc01fad1 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0510 expref: 785 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.803523:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f3:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.803528:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f3:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.803533:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b9880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993106176:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.803534:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0256e9 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.803551:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b2680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993105024:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.803562:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002cd7b47c/0xd5401cbabc01fad1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0510 expref: 785 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.803568:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0256c6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.803573:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57d80 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.803596:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002cd7b47c/0xd5401cbabc01fad1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0510 expref: 785 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.803611:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.803611:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9d180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993106304:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.803641:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470b40 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.803672:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.803693:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002390912 00000100:00100000:0.0:1783520935.803696:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002390912 00010000:00010000:0.0:1783520935.803715:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d208c8b1/0xd5401cbabc01fab5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04d8 expref: 786 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.803745:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002391040 00000100:00100000:0.0:1783520935.803750:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002391040 00000100:00100000:1.0:1783520935.803759:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35874 00010000:00010000:0.0:1783520935.803766:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f2004678/0xd5401cbabc01fabc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04e6 expref: 787 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.803773:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5bb800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+787:2388:x1870153002390912:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.803788:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.803795:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d208c8b1/0xd5401cbabc01fab5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ef:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04d8 expref: 787 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.803802:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35875 00000100:00100000:0.0:1783520935.803813:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638587c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+787:2388:x1870153002391040:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.803816:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d208c8b1/0xd5401cbabc01fab5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5ef:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04d8 expref: 787 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.803821:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.803828:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f2004678/0xd5401cbabc01fabc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04e6 expref: 787 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.803836:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d208c8b1/0xd5401cbabc01fab5 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5ef:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04d8 expref: 787 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.803839:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f2004678/0xd5401cbabc01fabc lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5f0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04e6 expref: 787 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.803863:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ef:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.803868:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ef:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.803868:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f2004678/0xd5401cbabc01fabc lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5f0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04e6 expref: 786 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.803878:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0256c6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.803882:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.803891:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.803893:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0256c6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.803897:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0256cd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.803912:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0256cd lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.803925:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.803934:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.803943:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0256c6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.803951:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0256c6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.803957:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0256c6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.803964:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0256c6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.803965:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5bb800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+785:2388:x1870153002390912:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 194us (352us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.803966:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638587c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+785:2388:x1870153002391040:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 151us (291us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.803985:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35874 00010000:00010000:2.0:1783520935.803989:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.803995:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35875 00010000:00010000:1.0:1783520935.804017:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0256cd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.804027:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0256cd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.804029:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.804036:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0256c6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.804040:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0256cd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.804043:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.804048:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0256c6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.804054:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0256cd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.804057:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0256c6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.804067:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515345500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+218:3625:x1870152993104896:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2327us (2446us total) trans 4294975782 rc 0/0 00010000:00010000:1.0:1783520935.804079:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520935.804082:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44283 00010000:00010000:0.0:1783520935.804084:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0256c6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.804091:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c42e258a/0xd5401cbabc0256c6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.804108:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.804112:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0256cd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.804119:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.804123:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0256cd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.804131:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0256cd lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.804143:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0256cd lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.804148:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c8b052a2/0xd5401cbabc0256cd lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.804167:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.804170:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.804173:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.804175:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350794f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+218:3625:x1870152993105152:12345-0@lo:6:osp-syn-1-0.0 Request processed in 1962us (2042us total) trans 4294975783 rc 0/0 00010000:00010000:0.0:1783520935.804179:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.804187:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44284 00000100:00100000:2.0:1783520935.804195:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.804204:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9df80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993105664:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.804223:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002391168 00010000:00010000:0.0:1783520935.804224:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0256db lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.804229:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002391168 00000004:00080000:1.0:1783520935.804230:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502253100/2, rc 0, transno 4294975783 00000100:00100000:1.0:1783520935.804247:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502253100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993105152:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.804251:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004c350f36/0xd5401cbabc01faca lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0502 expref: 784 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.804282:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35876 00000100:00100000:2.0:1783520935.804292:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5bb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+784:2387:x1870153002391168:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.804302:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.804304:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993106432:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.804311:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004c350f36/0xd5401cbabc01faca lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0502 expref: 784 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.804321:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004c350f36/0xd5401cbabc01faca lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5f2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0502 expref: 784 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.804338:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004c350f36/0xd5401cbabc01faca lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5f2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0502 expref: 784 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.804341:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.804352:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.804355:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.804384:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0256db lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.804389:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0256db lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.804391:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993106432 00000100:00100000:0.0:1783520935.804397:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993106432 00010000:00010000:2.0:1783520935.804408:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.804433:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44289 00000004:00080000:1.0:1783520935.804437:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515346d80/2, rc 0, transno 4294975782 00000100:00100000:2.0:1783520935.804440:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5bb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+783:2387:x1870153002391168:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 148us (244us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.804446:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+218:3627:x1870152993106432:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.804452:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515346d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993104896:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.804454:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35876 00002000:00080000:0.0:1783520935.804460:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1525 count 1 00010000:00010000:0.0:1783520935.804472:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000013affd72/0xd5401cbabc0256f0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5f5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.804482:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000618f05c0/0xd5401cbabc01fadf lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f052c expref: 782 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.804487:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f5:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.804490:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.804492:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000013affd72/0xd5401cbabc0256f0 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.804515:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:0.0:1783520935.804522:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000618f05c0/0xd5401cbabc01fadf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f052c expref: 782 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.804527:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993106560:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.804533:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000618f05c0/0xd5401cbabc01fadf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f052c expref: 782 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.804546:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9df80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993106688:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.804564:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.804593:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993106560 00000100:00100000:0.0:1783520935.804597:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470d80 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.804599:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993106560 00010000:00010000:0.0:1783520935.804625:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0256db lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.804634:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44290 00010000:00010000:0.0:1783520935.804636:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0256db lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.804647:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e1c7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+219:3627:x1870152993106560:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.804648:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0256db lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.804659:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0256db lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.804664:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1526 count 1 00010000:00010000:0.0:1783520935.804677:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.804679:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0256f7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5f6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.804687:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.804691:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000045ccf454/0xd5401cbabc022b99 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3289 expref: 782 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.804695:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0256db lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.804715:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfa000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.804720:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x5f6:0x0].0x0 since=36s v=(0 0) 00010000:00010000:1.0:1783520935.804721:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0256db lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.804725:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.804732:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0256f7 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.804749:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0256db lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.804759:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350794df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+220:3628:x1870152993105536:12345-0@lo:6:osp-syn-1-0.0 Request processed in 1825us (1895us total) trans 4294975784 rc 0/0 00010000:00010000:2.0:1783520935.804760:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000045ccf454/0xd5401cbabc022b99 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3289 expref: 782 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.804766:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0256db lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.804768:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44286 00010000:00010000:1.0:1783520935.804785:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c721a958/0xd5401cbabc0256db lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.804795:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000045ccf454/0xd5401cbabc022b99 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3289 expref: 782 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.804795:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b1500/2, rc 0, transno 4294975784 00010000:00010000:1.0:1783520935.804805:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.804805:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993105536:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.804810:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.804817:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.804822:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b9f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993106816:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.804854:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872f00 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.804870:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993106944:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.804901:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.805085:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993106944 00000100:00100000:1.0:1783520935.805090:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993106944 00000100:00100000:1.0:1783520935.805121:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44291 00000100:00100000:1.0:1783520935.805129:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+220:3625:x1870152993106944:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.805140:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1527 count 1 00010000:00010000:1.0:1783520935.805150:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0256fe lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5f7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.805158:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008664be3f/0xd5401cbabc01faed lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0548 expref: 782 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.805164:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f7:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.805166:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.805169:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0256fe lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.805187:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008664be3f/0xd5401cbabc01faed lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0548 expref: 782 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.805195:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008664be3f/0xd5401cbabc01faed lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0548 expref: 782 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.805203:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ae5c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993107072:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.805230:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9d180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993106304:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.805243:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6840 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.805254:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0256e9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.805291:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993106048:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.805306:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0256d4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.805349:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b9880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993106176:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.805380:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0256e2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.805432:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc022a50 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3140 expref: 782 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.805444:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc022a50 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3140 expref: 782 pid: 8410 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.805457:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000df839f65/0xd5401cbabc01fa5a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0430 expref: 782 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.805459:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993107200:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.805477:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c552d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993102080:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.805483:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.805513:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025672 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.805524:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025672 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.805528:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124706c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.805540:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025672 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.805550:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025672 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.805558:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025672 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.805569:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002391296 00000100:00100000:1.0:1783520935.805573:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002391296 00010000:00010000:2.0:1783520935.805574:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.805581:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.805588:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.805593:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002cd7b47c/0xd5401cbabc01fad1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0510 expref: 782 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.805596:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025672 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.805616:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5400 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520935.805616:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.805622:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025672 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.805627:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002391552 00010000:00010000:2.0:1783520935.805632:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025672 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.805632:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002391552 00010000:00010000:2.0:1783520935.805649:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025672 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.805653:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007872e47b/0xd5401cbabc01fad8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f051e expref: 783 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.805666:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025672 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.805682:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002391424 00010000:00010000:2.0:1783520935.805686:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.805687:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002391424 00010000:00010000:2.0:1783520935.805691:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.805695:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.805705:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c4d4ca72/0xd5401cbabc01fac3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04f4 expref: 784 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.805713:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35877 00000100:00100000:2.0:1783520935.805725:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ae5880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+784:2387:x1870153002391296:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.805736:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.805736:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35878 00010000:00010000:2.0:1783520935.805746:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002cd7b47c/0xd5401cbabc01fad1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0510 expref: 784 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.805751:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+784:2387:x1870153002391552:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.805766:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.805767:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002cd7b47c/0xd5401cbabc01fad1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5f3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0510 expref: 784 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.805774:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007872e47b/0xd5401cbabc01fad8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f051e expref: 784 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.805793:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002cd7b47c/0xd5401cbabc01fad1 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5f3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0510 expref: 784 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.805793:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007872e47b/0xd5401cbabc01fad8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5f4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f051e expref: 784 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.805817:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.805821:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007872e47b/0xd5401cbabc01fad8 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5f4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f051e expref: 783 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.805822:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.805832:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0256e9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.805839:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.805842:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0256e9 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.805844:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.805853:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0256e2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.805863:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0256e2 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.805871:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.805888:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.805907:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ae5880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+782:2387:x1870153002391296:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 182us (426us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.805915:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35877 00000100:00100000:1.0:1783520935.805921:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+782:2387:x1870153002391552:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 172us (342us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.805931:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35878 00000100:00100000:2.0:1783520935.805944:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35879 00000100:00100000:2.0:1783520935.805955:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9f800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+780:2387:x1870153002391424:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.805966:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.805966:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0256e2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.805974:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c4d4ca72/0xd5401cbabc01fac3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04f4 expref: 780 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.805979:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0256e2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.805993:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9df80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993106688:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.805995:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c4d4ca72/0xd5401cbabc01fac3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5f1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04f4 expref: 780 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.805996:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0256e2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.806011:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000013affd72/0xd5401cbabc0256f0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.806020:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0256e2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.806028:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c4d4ca72/0xd5401cbabc01fac3 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5f1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f04f4 expref: 780 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.806042:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.806045:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.806046:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.806053:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0256d4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.806061:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0256d4 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.806078:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.806100:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9f800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+779:2387:x1870153002391424:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 148us (488us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.806107:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35879 00000100:00100000:2.0:1783520935.806139:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b9f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993106816:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.806152:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0256f7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.806163:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+221:3626:x1870152993105920:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2812us (2896us total) trans 4294975785 rc 0/0 00000100:00100000:1.0:1783520935.806172:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44287 00000100:00100000:2.0:1783520935.806188:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000004:00080000:1.0:1783520935.806212:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9f480/2, rc 0, transno 4294975785 00000100:00100000:1.0:1783520935.806223:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993105920:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.806266:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ae7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993107328:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.806288:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.806304:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.806315:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.806339:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002391680 00000100:00100000:2.0:1783520935.806342:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002391680 00000100:00100000:1.0:1783520935.806353:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002391936 00000100:00100000:1.0:1783520935.806382:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002391936 00010000:00010000:2.0:1783520935.806384:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000618f05c0/0xd5401cbabc01fadf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f052c expref: 779 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.806404:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35880 00010000:00010000:1.0:1783520935.806404:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008664be3f/0xd5401cbabc01faed lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0548 expref: 780 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.806417:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5b9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+780:2388:x1870153002391680:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.806424:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002391808 00010000:00010000:2.0:1783520935.806428:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.806429:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002391808 00010000:00010000:2.0:1783520935.806436:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000618f05c0/0xd5401cbabc01fadf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f052c expref: 781 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.806449:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000045ccf454/0xd5401cbabc022b99 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3289 expref: 781 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.806451:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000618f05c0/0xd5401cbabc01fadf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5f5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f052c expref: 781 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.806482:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000618f05c0/0xd5401cbabc01fadf lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5f5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f052c expref: 781 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.806483:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35881 00000100:00100000:1.0:1783520935.806509:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5b9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+781:2388:x1870153002391936:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.806517:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.806521:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.806522:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.806531:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000013affd72/0xd5401cbabc0256f0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.806531:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008664be3f/0xd5401cbabc01faed lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5f7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0548 expref: 780 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.806558:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000013affd72/0xd5401cbabc0256f0 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.806565:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008664be3f/0xd5401cbabc01faed lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0548 expref: 780 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.806585:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.806585:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.806590:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.806595:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0256fe lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.806603:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0256fe lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5f7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.806612:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5b9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+780:2388:x1870153002391680:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 196us (424us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.806621:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.806622:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35880 00000100:00100000:2.0:1783520935.806643:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35882 00000100:00100000:1.0:1783520935.806648:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5b9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+779:2388:x1870153002391936:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 154us (359us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.806657:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638584850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+779:2390:x1870153002391808:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.806661:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35881 00010000:00010000:2.0:1783520935.806673:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.806680:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000045ccf454/0xd5401cbabc022b99 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5f6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3289 expref: 778 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.806692:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000045ccf454/0xd5401cbabc022b99 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5f6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3289 expref: 778 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.806703:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993107328 00000100:00100000:1.0:1783520935.806709:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993107328 00010000:00010000:2.0:1783520935.806710:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000045ccf454/0xd5401cbabc022b99 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5f6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3289 expref: 778 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.806722:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.806729:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.806730:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0256e2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.806735:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f6:0x0].0x0 since=0s v=(1 0) 00000100:00100000:1.0:1783520935.806744:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44292 00010000:00010000:2.0:1783520935.806745:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0256f7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.806758:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ae7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+221:3626:x1870152993107328:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.806768:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.806769:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0256f7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.806776:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0256e2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.806781:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1528 count 1 00010000:00010000:0.0:1783520935.806790:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0256e2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.806791:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.806799:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025705 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5f8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.806808:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0256e2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.806815:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000023ec40c2/0xd5401cbabc01faf4 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0556 expref: 777 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.806816:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638584850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+777:2390:x1870153002391808:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 160us (498us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.806829:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0256e2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.806835:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35882 00010000:00010000:1.0:1783520935.806854:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f8:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.806858:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.806862:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025705 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.806868:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.806872:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.806875:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.806881:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0256f7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.806886:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000023ec40c2/0xd5401cbabc01faf4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0556 expref: 776 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.806893:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0256f7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.806893:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000013affd72/0xd5401cbabc0256f0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.806904:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000023ec40c2/0xd5401cbabc01faf4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0556 expref: 776 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.806915:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0256f7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.806919:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000013affd72/0xd5401cbabc0256f0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.806933:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ae5180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993107456:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.806941:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0256f7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.806946:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000013affd72/0xd5401cbabc0256f0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.806964:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000013affd72/0xd5401cbabc0256f0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.806967:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.806982:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520935.806984:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6cc0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.806987:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0256f7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.806996:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.807000:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0256f7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.807009:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0256f7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.807014:0:12734:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008664be3f/0xd5401cbabc01faed lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0548 expref: 776 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.807027:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ae5c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993107072:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.807045:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0256f7 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.807051:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0256fe lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.807053:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0256f7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.807061:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e1c7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+222:3627:x1870152993106560:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2418us (2497us total) trans 4294975786 rc 0/0 00000100:00100000:2.0:1783520935.807076:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44290 00010000:00010000:0.0:1783520935.807091:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.807096:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520935.807097:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e1c7480/2, rc 0, transno 4294975786 00010000:00010000:0.0:1783520935.807101:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.807104:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0256e9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.807110:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0256e9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.807111:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993106560:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.807119:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.807127:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0256e9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.807132:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0256e9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.807143:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520935.807150:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520935.807154:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000013affd72/0xd5401cbabc0256f0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.807158:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf7c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.807161:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000013affd72/0xd5401cbabc0256f0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.807166:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000013affd72/0xd5401cbabc0256f0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.807174:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.807179:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000013affd72/0xd5401cbabc0256f0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.807181:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0256e9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.807186:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000013affd72/0xd5401cbabc0256f0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.807198:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfa600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.807204:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0256e9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.807209:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.807213:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.807213:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0256e9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.807216:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.807222:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+222:3627:x1870152993106432:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2779us (2882us total) trans 4294975787 rc 0/0 00010000:00010000:1.0:1783520935.807233:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0256e9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.807233:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44289 00010000:00010000:1.0:1783520935.807244:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e1c39107/0xd5401cbabc0256e9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.807260:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:1.0:1783520935.807266:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.807270:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.807273:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.807279:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ae5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993107584:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.807299:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+222:3626:x1870152993105792:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3860us (3957us total) trans 4294975788 rc 0/0 00000004:00080000:1.0:1783520935.807301:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b2680/2, rc 0, transno 4294975787 00000100:00100000:2.0:1783520935.807307:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44288 00000100:00100000:1.0:1783520935.807315:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993106432:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.807324:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.807386:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ae6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993107712:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.807421:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.807428:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:1.0:1783520935.807441:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9ed80/2, rc 0, transno 4294975788 00000100:00100000:1.0:1783520935.807456:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993105792:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.807486:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993107200:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.807494:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1783520935.807519:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ae7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993107840:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.807521:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025687 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.807547:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993107584 00000100:00100000:0.0:1783520935.807552:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993107584 00000100:00100000:1.0:1783520935.807557:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.807583:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993107712 00000100:00100000:0.0:1783520935.807587:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993107712 00000100:00100000:1.0:1783520935.807605:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993107840 00000100:00100000:1.0:1783520935.807610:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993107840 00000100:00100000:0.0:1783520935.807617:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44293 00000100:00100000:2.0:1783520935.807623:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+225:3625:x1870152993101824:12345-0@lo:6:osp-syn-1-0.0 Request processed in 11579us (11676us total) trans 4294975789 rc 0/0 00000100:00100000:0.0:1783520935.807631:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+225:3628:x1870152993107584:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.807636:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44271 00000100:00100000:1.0:1783520935.807643:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44294 00002000:00080000:0.0:1783520935.807649:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1529 count 1 00000100:00100000:1.0:1783520935.807656:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+224:3628:x1870152993107712:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.807664:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44295 00010000:00010000:0.0:1783520935.807665:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df5f0644/0xd5401cbabc02570c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5f9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.807672:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1530 count 1 00000100:00100000:2.0:1783520935.807678:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ae4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+224:3626:x1870152993107840:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.807680:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000031551624/0xd5401cbabc01fafb lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0564 expref: 775 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.807689:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc025713 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5fa:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.807691:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f9:0x0].0x0 since=37s v=(0 0) 00002000:00080000:2.0:1783520935.807695:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1531 count 1 00010000:00010000:0.0:1783520935.807701:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.807711:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df5f0644/0xd5401cbabc02570c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.807713:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02571a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5fb:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.807715:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000586a94ed/0xd5401cbabc01fb02 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0572 expref: 775 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.807739:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5fa:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.807744:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ac4a8ebc/0xd5401cbabc01fb09 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0580 expref: 775 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.807745:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5fa:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.807755:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000031551624/0xd5401cbabc01fafb lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0564 expref: 775 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.807756:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc025713 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.807784:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5fb:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.807787:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000031551624/0xd5401cbabc01fafb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0564 expref: 775 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.807789:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5fb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.807798:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02571a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.807800:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000586a94ed/0xd5401cbabc01fb02 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0572 expref: 775 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.807812:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993107968:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.807829:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000586a94ed/0xd5401cbabc01fb02 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0572 expref: 775 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.807845:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ae4700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993108096:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.807846:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ac4a8ebc/0xd5401cbabc01fb09 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0580 expref: 775 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.807861:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ac4a8ebc/0xd5401cbabc01fb09 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0580 expref: 775 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.807869:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124706c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.807875:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b9f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993108224:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.807882:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6840 going to sleep for 11 seconds 00000004:00080000:1.0:1783520935.807909:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435079f0000/2, rc 0, transno 4294975789 00000100:00100000:2.0:1783520935.807910:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435131fc3c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.807918:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993101824:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.807956:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1783520935.807964:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ae5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993108352:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.807995:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.807996:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0256d4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.808006:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0256d4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.808015:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0256d4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.808019:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993108352 00000100:00100000:2.0:1783520935.808025:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993108352 00000100:00100000:1.0:1783520935.808026:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002392064 00010000:00010000:0.0:1783520935.808026:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0256d4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.808033:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002392064 00010000:00010000:0.0:1783520935.808048:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.808055:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc022a50 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3140 expref: 776 pid: 8410 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.808058:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:2.0:1783520935.808060:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44296 00010000:00010000:0.0:1783520935.808065:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0256d4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.808075:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507acf480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+225:3625:x1870152993108352:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.808089:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbfb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.808093:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0256d4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.808094:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1532 count 1 00000100:00100000:1.0:1783520935.808094:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35883 00000100:00100000:1.0:1783520935.808106:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5bb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+776:2388:x1870153002392064:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.808108:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0256d4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.808111:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025721 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5fc:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.808114:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.808121:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc022a50 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5e6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3140 expref: 776 pid: 8410 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.808131:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0256d4 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.808141:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000610cd0d0/0xd5401cbabc01fb10 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f058e expref: 776 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.808144:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc022a50 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5e6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3140 expref: 776 pid: 8410 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.808149:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000381573cd/0xd5401cbabc0256d4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.808173:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc022a50 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5e6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3140 expref: 776 pid: 8410 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.808183:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.808185:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5fc:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.808188:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.808190:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5fc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.808191:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.808192:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.808197:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5e6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.808198:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025721 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.808201:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025687 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.808217:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025687 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.808233:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000610cd0d0/0xd5401cbabc01fb10 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f058e expref: 775 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.808235:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.808247:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000610cd0d0/0xd5401cbabc01fb10 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f058e expref: 775 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.808260:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b9c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993108480:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.808262:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5bb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+775:2388:x1870153002392064:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 157us (840us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.808270:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35883 00000100:00100000:2.0:1783520935.808297:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435131fc300 going to sleep for 11 seconds 00000004:00080000:1.0:1783520935.808300:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350794e300/2, rc 0, transno 4294975790 00000100:00100000:0.0:1783520935.808304:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5baa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+226:3627:x1870152993105408:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5583us (5706us total) trans 4294975790 rc 0/0 00000100:00100000:1.0:1783520935.808311:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350794e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993105408:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.808316:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44285 00010000:00010000:0.0:1783520935.808343:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025687 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.808353:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025687 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.808374:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507acf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993108608:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.808390:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025687 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.808397:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025687 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.808408:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.808412:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.808423:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.808427:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025687 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.808436:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5e00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520935.808439:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.808442:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025687 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.808452:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025687 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.808462:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002392192 00010000:00010000:0.0:1783520935.808466:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025687 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.808468:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002392192 00010000:00010000:0.0:1783520935.808478:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025687 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.808489:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000023ec40c2/0xd5401cbabc01faf4 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0556 expref: 775 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.808493:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.808511:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.808515:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.808532:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35884 00000100:00100000:1.0:1783520935.808540:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5b8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+775:2387:x1870153002392192:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.808547:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.808550:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000023ec40c2/0xd5401cbabc01faf4 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x5f8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0556 expref: 775 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.808558:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000023ec40c2/0xd5401cbabc01faf4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0556 expref: 775 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.808568:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.808570:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.808574:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025705 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.808580:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025705 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5f8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.808595:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.808615:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+226:3627:x1870152993102720:12345-0@lo:6:osp-syn-1-0.0 Request processed in 11298us (11625us total) trans 4294975791 rc 0/0 00000100:00100000:0.0:1783520935.808624:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44274 00000004:00080000:2.0:1783520935.808632:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b2300/2, rc 0, transno 4294975791 00000100:00100000:1.0:1783520935.808632:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5b8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+775:2387:x1870153002392192:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 90us (194us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.808644:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35884 00000100:00100000:0.0:1783520935.808646:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993108608 00000100:00100000:2.0:1783520935.808649:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993102720:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.808652:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993108608 00000100:00100000:0.0:1783520935.808687:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44297 00000100:00100000:0.0:1783520935.808696:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635601450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+226:3627:x1870152993108608:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.808706:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00002000:00080000:0.0:1783520935.808710:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1533 count 1 00000100:00100000:2.0:1783520935.808718:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993108736:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.808723:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc025728 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5fd:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.808727:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0256fe lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.808739:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0256fe lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.808741:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009f04bbe5/0xd5401cbabc01fb17 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f059c expref: 774 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.808752:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0256fe lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.808756:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.808764:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5fd:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.808766:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0256fe lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.808770:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5fd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.808780:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc025728 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.808787:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.808792:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.808798:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0256fe lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.808803:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009f04bbe5/0xd5401cbabc01fb17 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f059c expref: 774 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.808813:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.808821:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0256fe lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.808823:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009f04bbe5/0xd5401cbabc01fb17 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f059c expref: 774 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.808839:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0256fe lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.808861:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9d180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993108864:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.808865:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0256fe lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.808874:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4fdb58b/0xd5401cbabc0256fe lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.808884:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+227:3625:x1870152993106944:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3753us (3979us total) trans 4294975792 rc 0/0 00000100:00100000:1.0:1783520935.808894:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44291 00010000:00010000:2.0:1783520935.808899:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.808904:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.808908:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.808921:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993108736 00000100:00100000:0.0:1783520935.808922:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470b40 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.808928:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993108736 00000100:00100000:0.0:1783520935.808959:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44298 00000100:00100000:0.0:1783520935.808969:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5b9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+227:3627:x1870152993108736:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.808982:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1534 count 1 00000004:00080000:1.0:1783520935.808993:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e1c6d80/2, rc 0, transno 4294975792 00010000:00010000:0.0:1783520935.808995:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000d75043e/0xd5401cbabc02572f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5fe:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.809010:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993106944:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.809011:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001080b2fc/0xd5401cbabc022a88 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5fe:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3178 expref: 774 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.809022:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x5fe:0x0].0x0 since=36s v=(0 0) 00010000:00010000:0.0:1783520935.809026:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5fe:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.809030:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000d75043e/0xd5401cbabc02572f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5fe:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.809045:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000023ec40c2/0xd5401cbabc01faf4 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0556 expref: 774 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.809053:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:0.0:1783520935.809058:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001080b2fc/0xd5401cbabc022a88 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5fe:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3178 expref: 773 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.809061:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ae5180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993107456:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.809066:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993108992:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.809082:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001080b2fc/0xd5401cbabc022a88 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5fe:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3178 expref: 773 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.809087:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025705 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.809095:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9f800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993109120:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.809100:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025705 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.809104:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.809113:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025705 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.809122:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025705 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.809127:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470cc0 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.809134:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025705 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.809145:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993108992 00000100:00100000:0.0:1783520935.809149:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993108992 00010000:00010000:1.0:1783520935.809155:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.809166:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.809173:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025705 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.809173:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44299 00000100:00100000:0.0:1783520935.809183:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5b9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+228:3626:x1870152993108992:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.809187:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.809192:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025705 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.809198:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1535 count 1 00010000:00010000:2.0:1783520935.809204:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025705 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.809212:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005d2d45eb/0xd5401cbabc025736 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5ff:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.809219:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025705 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.809226:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008abce240/0xd5401cbabc01fb25 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x5ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f05b8 expref: 773 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.809232:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e3859524/0xd5401cbabc025705 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.809240:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ff:0x0].0x0 since=37s v=(0 0) 00000100:00100000:1.0:1783520935.809247:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ae7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+229:3626:x1870152993107328:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2490us (2941us total) trans 4294975793 rc 0/0 00010000:00010000:0.0:1783520935.809247:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ff:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.809255:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005d2d45eb/0xd5401cbabc025736 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x5ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.809257:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.809261:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44292 00010000:00010000:2.0:1783520935.809263:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.809267:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.809279:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008abce240/0xd5401cbabc01fb25 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05b8 expref: 773 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.809286:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008abce240/0xd5401cbabc01fb25 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05b8 expref: 773 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.809294:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a4700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993109248:192.168.204.33@tcp:104: 00000004:00080000:1.0:1783520935.809316:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ae7800/2, rc 0, transno 4294975793 00000100:00100000:0.0:1783520935.809324:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.809334:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ae7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993107328:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.809399:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993107968:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.809415:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000df5f0644/0xd5401cbabc02570c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.809427:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1783520935.809434:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b9f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993108224:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.809439:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993109376:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.809453:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02571a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.809463:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.809481:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.809485:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002392448 00000100:00100000:2.0:1783520935.809490:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002392448 00000100:00100000:0.0:1783520935.809493:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.809524:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000031551624/0xd5401cbabc01fafb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0564 expref: 774 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.809542:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.809549:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ae4700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993108096:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.809553:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002392320 00000100:00100000:2.0:1783520935.809558:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002392320 00010000:00010000:1.0:1783520935.809563:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc025713 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.809576:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ac4a8ebc/0xd5401cbabc01fb09 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0580 expref: 775 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.809582:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002392576 00000100:00100000:1.0:1783520935.809587:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002392576 00000100:00100000:2.0:1783520935.809604:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35885 00010000:00010000:1.0:1783520935.809608:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000586a94ed/0xd5401cbabc01fb02 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0572 expref: 776 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.809614:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993109376 00000100:00100000:2.0:1783520935.809619:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+776:2387:x1870153002392448:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.809621:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993109376 00010000:00010000:2.0:1783520935.809632:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.809638:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000031551624/0xd5401cbabc01fafb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5f9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0564 expref: 776 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.809642:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35886 00000100:00100000:0.0:1783520935.809652:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44300 00000100:00100000:1.0:1783520935.809656:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+776:2387:x1870153002392320:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.809658:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000031551624/0xd5401cbabc01fafb lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5f9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0564 expref: 776 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.809663:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507acf100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+229:3625:x1870152993109376:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.809666:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.809674:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ac4a8ebc/0xd5401cbabc01fb09 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5fb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0580 expref: 776 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.809680:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1536 count 1 00010000:00010000:2.0:1783520935.809686:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000031551624/0xd5401cbabc01fafb lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5f9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0564 expref: 776 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.809687:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ac4a8ebc/0xd5401cbabc01fb09 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5fb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0580 expref: 776 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.809695:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e23a9c85/0xd5401cbabc02573d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x600:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.809716:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.809719:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a4a746ba/0xd5401cbabc01fb33 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x600:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f05c6 expref: 775 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.809722:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ac4a8ebc/0xd5401cbabc01fb09 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5fb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0580 expref: 775 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.809723:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5f9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.809734:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000df5f0644/0xd5401cbabc02570c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.809741:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x600:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.809749:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x600:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.809757:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e23a9c85/0xd5401cbabc02573d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x600:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.809760:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df5f0644/0xd5401cbabc02570c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.809770:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5fb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.809778:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5fb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.809784:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02571a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.809791:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a4a746ba/0xd5401cbabc01fb33 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x600:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05c6 expref: 774 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.809794:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.809795:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02571a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.809814:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a4a746ba/0xd5401cbabc01fb33 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x600:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05c6 expref: 774 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.809820:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+774:2387:x1870153002392448:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 205us (357us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.809827:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.809835:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35885 00000100:00100000:0.0:1783520935.809836:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530e300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993109504:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.809857:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35887 00000100:00100000:1.0:1783520935.809857:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+773:2387:x1870153002392320:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 202us (363us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.809872:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35886 00000100:00100000:0.0:1783520935.809872:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124706c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.809873:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530c700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+773:2387:x1870153002392576:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.809884:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.809891:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000586a94ed/0xd5401cbabc01fb02 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5fa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0572 expref: 772 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.809902:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000586a94ed/0xd5401cbabc01fb02 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5fa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0572 expref: 772 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.809916:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000586a94ed/0xd5401cbabc01fb02 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5fa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0572 expref: 772 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.809926:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5fa:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.809930:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5fa:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.809934:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc025713 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.809942:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc025713 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.809960:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.809974:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc025713 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.809982:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530c700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+771:2387:x1870153002392576:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 113us (439us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.809989:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc025713 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.809994:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35887 00010000:00010000:1.0:1783520935.810004:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc025713 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.810011:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc025713 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.810021:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.810029:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000df5f0644/0xd5401cbabc02570c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.810031:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.810043:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000df5f0644/0xd5401cbabc02570c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.810044:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.810051:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc025713 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.810072:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.810074:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df5f0644/0xd5401cbabc02570c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.810077:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc025713 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.810090:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000df5f0644/0xd5401cbabc02570c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.810094:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc025713 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.810107:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.810110:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc025713 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.810119:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005f11335d/0xd5401cbabc025713 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.810125:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+230:3628:x1870152993107712:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2473us (2699us total) trans 4294975794 rc 0/0 00000100:00100000:1.0:1783520935.810136:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44294 00010000:00010000:0.0:1783520935.810148:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.810153:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000df5f0644/0xd5401cbabc02570c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.810160:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf6400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.810164:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000df5f0644/0xd5401cbabc02570c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.810167:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ae6300/2, rc 0, transno 4294975794 00010000:00010000:0.0:1783520935.810176:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000df5f0644/0xd5401cbabc02570c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.810180:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ae6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993107712:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.810190:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+230:3628:x1870152993107584:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2563us (2868us total) trans 4294975795 rc 0/0 00010000:00010000:0.0:1783520935.810195:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000df5f0644/0xd5401cbabc02570c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.810196:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ae5c00/2, rc 0, transno 4294975795 00000100:00100000:2.0:1783520935.810201:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44293 00000100:00100000:1.0:1783520935.810204:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ae5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993107584:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.810214:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000df5f0644/0xd5401cbabc02570c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.810235:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.810239:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.810241:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.810249:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b9c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993108480:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.810252:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.810264:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ace300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993109632:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.810266:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025721 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.810303:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.810335:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1783520935.810340:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9d180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993108864:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.810346:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507acce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993109760:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.810383:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc025728 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.810403:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02571a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.810409:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.810417:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02571a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.810427:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02571a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.810431:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.810440:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02571a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.810450:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002392704 00000100:00100000:1.0:1783520935.810455:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002392704 00010000:00010000:2.0:1783520935.810457:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.810467:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.810471:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02571a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.810475:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000610cd0d0/0xd5401cbabc01fb10 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f058e expref: 771 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.810485:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.810490:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.810509:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02571a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.810519:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02571a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.810523:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002392832 00000100:00100000:1.0:1783520935.810528:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002392832 00010000:00010000:2.0:1783520935.810536:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02571a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.810545:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02571a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.810547:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009f04bbe5/0xd5401cbabc01fb17 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f059c expref: 772 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.810562:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a4700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993109248:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.810583:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.810584:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002392960 00010000:00010000:0.0:1783520935.810585:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005d2d45eb/0xd5401cbabc025736 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x5ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.810587:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.810589:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002392960 00010000:00010000:2.0:1783520935.810591:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.810610:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001080b2fc/0xd5401cbabc022a88 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x5fe:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3178 expref: 773 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.810611:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35888 00000100:00100000:0.0:1783520935.810618:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993109632 00000100:00100000:0.0:1783520935.810624:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993109632 00000100:00100000:2.0:1783520935.810626:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530fb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+773:2387:x1870153002392704:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.810643:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.810650:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000610cd0d0/0xd5401cbabc01fb10 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5fc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f058e expref: 773 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.810657:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993109760 00000100:00100000:1.0:1783520935.810661:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35889 00000100:00100000:0.0:1783520935.810662:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993109760 00010000:00010000:2.0:1783520935.810665:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000610cd0d0/0xd5401cbabc01fb10 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5fc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f058e expref: 773 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.810675:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638587850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+773:2388:x1870153002392832:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.810681:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000610cd0d0/0xd5401cbabc01fb10 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5fc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f058e expref: 773 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.810686:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.810692:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44301 00010000:00010000:1.0:1783520935.810696:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009f04bbe5/0xd5401cbabc01fb17 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5fd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f059c expref: 772 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.810697:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5fc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.810706:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5fc:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.810706:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507acc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+231:3627:x1870152993109632:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.810716:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025721 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.810723:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009f04bbe5/0xd5401cbabc01fb17 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5fd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f059c expref: 772 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.810726:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1537 count 1 00010000:00010000:2.0:1783520935.810727:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025721 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.810742:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000013dcb818/0xd5401cbabc025744 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x601:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.810753:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009f04bbe5/0xd5401cbabc01fb17 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5fd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f059c expref: 772 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.810756:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.810758:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000033ca9bdc/0xd5401cbabc01fb2c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x601:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f05d4 expref: 772 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.810775:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x601:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.810778:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5fd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.810779:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x601:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.810783:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530fb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+771:2387:x1870153002392704:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 160us (761us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.810787:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5fd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.810789:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000013dcb818/0xd5401cbabc025744 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x601:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.810794:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35888 00010000:00010000:1.0:1783520935.810796:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc025728 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.810813:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc025728 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.810814:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35890 00010000:00010000:0.0:1783520935.810825:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000033ca9bdc/0xd5401cbabc01fb2c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x601:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05d4 expref: 770 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.810829:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530ca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+770:2389:x1870153002392960:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.810839:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.810845:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.810845:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000033ca9bdc/0xd5401cbabc01fb2c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x601:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05d4 expref: 770 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.810855:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001080b2fc/0xd5401cbabc022a88 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x5fe:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3178 expref: 770 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.810863:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530f480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993109888:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.810865:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638587850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+770:2388:x1870153002392832:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 192us (434us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.810882:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35889 00010000:00010000:2.0:1783520935.810888:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001080b2fc/0xd5401cbabc022a88 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x5fe:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3178 expref: 770 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.810901:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5fe:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.810904:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5fe:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.810907:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000000d75043e/0xd5401cbabc02572f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5fe:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.810913:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000d75043e/0xd5401cbabc02572f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5fe:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.810924:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.810928:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.810957:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530ca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+769:2389:x1870153002392960:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 130us (473us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.810958:0:35282:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001080b2fc/0xd5401cbabc022a88 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5fe:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3178 expref: 769 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.810961:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44302 00000100:00100000:1.0:1783520935.810976:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507acd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+231:3627:x1870152993109760:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.810978:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35890 00000100:00100000:0.0:1783520935.810985:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9f800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993109120:192.168.204.33@tcp:104: 00002000:00080000:1.0:1783520935.810995:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1538 count 1 00010000:00010000:0.0:1783520935.811005:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000000d75043e/0xd5401cbabc02572f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.811010:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025721 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.811011:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc02574b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x602:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.811018:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000000d75043e/0xd5401cbabc02572f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.811036:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025721 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.811038:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000000d75043e/0xd5401cbabc02572f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.811042:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005310f324/0xd5401cbabc01fb3a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x602:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f05e2 expref: 767 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.811061:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025721 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.811065:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000d75043e/0xd5401cbabc02572f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.811066:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x602:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.811075:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x602:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.811084:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025721 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.811086:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc02574b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x602:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.811087:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000000d75043e/0xd5401cbabc02572f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.811117:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:0.0:1783520935.811123:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520935.811126:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1783520935.811130:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005310f324/0xd5401cbabc01fb3a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x602:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05e2 expref: 767 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.811131:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025721 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.811134:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.811142:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000000d75043e/0xd5401cbabc02572f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.811152:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.811157:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025721 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.811160:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005310f324/0xd5401cbabc01fb3a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x602:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05e2 expref: 767 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.811160:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf6200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.811168:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000000d75043e/0xd5401cbabc02572f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.811171:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025721 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.811183:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507acea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993110016:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.811194:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000000d75043e/0xd5401cbabc02572f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.811198:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025721 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.811210:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025721 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.811213:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000000d75043e/0xd5401cbabc02572f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.811217:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6cc0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.811228:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000d75043e/0xd5401cbabc02572f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.811235:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.811239:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.811243:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.811248:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.811251:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.811255:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.811265:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc025728 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.811273:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc025728 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.811282:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc025728 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.811289:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc025728 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.811311:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.811324:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.811331:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc025728 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.811345:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507acf480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+232:3625:x1870152993108352:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3273us (3346us total) trans 4294975796 rc 0/0 00010000:00010000:1.0:1783520935.811349:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf6a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.811356:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc025728 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.811380:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44296 00010000:00010000:1.0:1783520935.811390:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc025728 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.811402:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc025728 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.811407:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f62dd5cc/0xd5401cbabc025728 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.811429:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635601450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+232:3627:x1870152993108608:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2732us (3018us total) trans 4294975797 rc 0/0 00000100:00100000:2.0:1783520935.811438:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.811443:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44297 00010000:00010000:1.0:1783520935.811446:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.811449:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.811452:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520935.811478:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507acf800/2, rc 0, transno 4294975797 00000100:00100000:1.0:1783520935.811491:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507acf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993108608:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.811558:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507acc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993110144:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.811578:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.811580:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5b9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+232:3627:x1870152993108736:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2610us (2825us total) trans 4294975798 rc 0/0 00000100:00100000:0.0:1783520935.811590:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44298 00000100:00100000:1.0:1783520935.811597:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.811619:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993110144 00000100:00100000:0.0:1783520935.811623:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993110144 00000004:00080000:1.0:1783520935.811640:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ae5880/2, rc 0, transno 4294975796 00000100:00100000:0.0:1783520935.811651:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44303 00000100:00100000:1.0:1783520935.811653:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ae5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993108352:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.811665:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ae4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+232:3625:x1870152993110144:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.811680:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1539 count 1 00010000:00010000:0.0:1783520935.811696:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003680c3ad/0xd5401cbabc025752 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x603:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.811706:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c5b9500/2, rc 0, transno 4294975798 00010000:00010000:0.0:1783520935.811712:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ee13ff38/0xd5401cbabc01fb41 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x603:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f05f0 expref: 767 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.811719:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993108736:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.811724:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x603:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.811729:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x603:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.811734:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003680c3ad/0xd5401cbabc025752 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x603:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.811748:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ae4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+233:3626:x1870152993107840:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4073us (4192us total) trans 4294975799 rc 0/0 00000100:00100000:2.0:1783520935.811756:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44295 00010000:00010000:0.0:1783520935.811760:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ee13ff38/0xd5401cbabc01fb41 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x603:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05f0 expref: 767 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.811762:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1783520935.811775:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350793b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993110400:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.811790:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ee13ff38/0xd5401cbabc01fb41 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x603:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05f0 expref: 767 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.811803:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993110528:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.811813:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.811866:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470b40 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.811872:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ae5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993110272:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.811898:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.811902:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530e300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993109504:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.811924:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e23a9c85/0xd5401cbabc02573d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x600:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.811925:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993110400 00000004:00080000:1.0:1783520935.811929:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ae7480/2, rc 0, transno 4294975799 00000100:00100000:2.0:1783520935.811932:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993110400 00000100:00100000:1.0:1783520935.811944:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ae7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993107840:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.811958:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993110272 00000100:00100000:2.0:1783520935.811962:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993110272 00000100:00100000:2.0:1783520935.811983:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44304 00000100:00100000:2.0:1783520935.811991:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350793aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+234:3628:x1870152993110400:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.812000:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507939f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993110656:0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.812002:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1541 count 1 00010000:00010000:2.0:1783520935.812015:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025759 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x605:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.812021:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44305 00010000:00010000:2.0:1783520935.812027:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000499c5659/0xd5401cbabc01fb4f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x605:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f060c expref: 767 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.812034:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507939180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+234:3626:x1870152993110272:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.812035:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x605:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.812043:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x605:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.812043:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.812048:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025759 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x605:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.812054:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1540 count 1 00010000:00010000:2.0:1783520935.812066:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000499c5659/0xd5401cbabc01fb4f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x605:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f060c expref: 767 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.812068:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000004b9eec72/0xd5401cbabc025760 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x604:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.812078:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000499c5659/0xd5401cbabc01fb4f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x605:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f060c expref: 767 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.812079:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002393088 00000100:00100000:1.0:1783520935.812084:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002393088 00010000:00010000:0.0:1783520935.812087:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000817fcacc/0xd5401cbabc01fb48 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x604:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f05fe expref: 767 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.812101:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008abce240/0xd5401cbabc01fb25 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05b8 expref: 768 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.812113:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x604:0x0].0x0 since=37s v=(0 0) 00000100:00100000:2.0:1783520935.812115:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5baa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993110784:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.812119:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x604:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.812124:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000004b9eec72/0xd5401cbabc025760 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x604:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.812127:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002393216 00000100:00100000:1.0:1783520935.812133:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002393216 00000100:00100000:2.0:1783520935.812146:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435131fc300 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.812155:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a4a746ba/0xd5401cbabc01fb33 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x600:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05c6 expref: 769 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.812161:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993110656 00010000:00010000:0.0:1783520935.812161:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000817fcacc/0xd5401cbabc01fb48 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x604:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05fe expref: 769 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.812167:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993110656 00010000:00010000:0.0:1783520935.812179:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000817fcacc/0xd5401cbabc01fb48 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x604:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05fe expref: 769 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.812193:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530e300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993110912:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.812198:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44306 00000100:00100000:1.0:1783520935.812198:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35891 00000100:00100000:2.0:1783520935.812209:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507938380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+235:3628:x1870152993110656:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.812213:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5b9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+769:2388:x1870153002393088:12345-192.168.204.33@tcp:103:ldlm_bl.0 00002000:00080000:2.0:1783520935.812225:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1542 count 1 00010000:00010000:1.0:1783520935.812225:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.812231:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124706c0 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.812234:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008abce240/0xd5401cbabc01fb25 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x5ff:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05b8 expref: 769 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.812241:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025767 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x606:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.812254:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008abce240/0xd5401cbabc01fb25 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x5ff:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05b8 expref: 769 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.812258:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000fd59ddf1/0xd5401cbabc022977 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x606:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3067 expref: 769 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.812268:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x606:0x0].0x0 since=36s v=(0 0) 00010000:00010000:1.0:1783520935.812270:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008abce240/0xd5401cbabc01fb25 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x5ff:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05b8 expref: 769 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.812273:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x606:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.812282:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025767 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x606:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.812288:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ff:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.812295:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x5ff:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.812300:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005d2d45eb/0xd5401cbabc025736 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.812310:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005d2d45eb/0xd5401cbabc025736 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x5ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.812311:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000fd59ddf1/0xd5401cbabc022977 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x606:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3067 expref: 768 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.812335:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000fd59ddf1/0xd5401cbabc022977 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x606:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3067 expref: 768 pid: 9151 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.812347:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.812378:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5bb480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993111040:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.812378:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005d2d45eb/0xd5401cbabc025736 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.812392:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5b9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+768:2388:x1870153002393088:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 182us (958us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.812394:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005d2d45eb/0xd5401cbabc025736 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.812401:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35891 00010000:00010000:0.0:1783520935.812410:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005d2d45eb/0xd5401cbabc025736 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x5ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.812418:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005d2d45eb/0xd5401cbabc025736 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.812422:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435131fc3c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.812425:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35892 00000100:00100000:1.0:1783520935.812439:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5b9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+767:2387:x1870153002393216:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.812448:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.812448:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.812450:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.812458:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005d2d45eb/0xd5401cbabc025736 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.812458:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a4a746ba/0xd5401cbabc01fb33 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x600:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05c6 expref: 767 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.812477:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf7400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.812483:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005d2d45eb/0xd5401cbabc025736 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x5ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.812494:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005d2d45eb/0xd5401cbabc025736 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x5ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.812509:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a4a746ba/0xd5401cbabc01fb33 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x600:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05c6 expref: 767 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.812524:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005d2d45eb/0xd5401cbabc025736 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x5ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.812532:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005d2d45eb/0xd5401cbabc025736 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x5ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.812533:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a4a746ba/0xd5401cbabc01fb33 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x600:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05c6 expref: 767 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.812558:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x600:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.812561:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.812562:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x600:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.812564:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.812566:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5b9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+236:3626:x1870152993108992:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3382us (3461us total) trans 4294975800 rc 0/0 00010000:00010000:2.0:1783520935.812567:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.812571:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e23a9c85/0xd5401cbabc02573d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x600:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.812579:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44299 00010000:00010000:1.0:1783520935.812586:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e23a9c85/0xd5401cbabc02573d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x600:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.812612:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.812618:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530f480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993109888:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.812634:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000013dcb818/0xd5401cbabc025744 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x601:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.812642:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5b9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+766:2387:x1870153002393216:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 203us (1064us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.812654:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35892 00010000:00010000:0.0:1783520935.812661:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e23a9c85/0xd5401cbabc02573d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x600:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.812670:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e23a9c85/0xd5401cbabc02573d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x600:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.812678:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e23a9c85/0xd5401cbabc02573d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x600:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.812686:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e23a9c85/0xd5401cbabc02573d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x600:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.812696:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c5ba680/2, rc 0, transno 4294975800 00010000:00010000:0.0:1783520935.812705:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.812709:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520935.812710:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993108992:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.812717:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e23a9c85/0xd5401cbabc02573d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x600:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.812732:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf7800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.812736:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e23a9c85/0xd5401cbabc02573d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x600:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.812745:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e23a9c85/0xd5401cbabc02573d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x600:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.812754:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507938e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993111168:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.812757:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e23a9c85/0xd5401cbabc02573d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x600:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.812766:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e23a9c85/0xd5401cbabc02573d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x600:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.812786:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.812790:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.812793:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.812793:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507acf100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+236:3625:x1870152993109376:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3132us (3310us total) trans 4294975801 rc 0/0 00000100:00100000:1.0:1783520935.812801:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.812809:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44300 00000100:00100000:0.0:1783520935.812830:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993111168 00000100:00100000:0.0:1783520935.812835:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993111168 00000004:00080000:1.0:1783520935.812843:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c5ba300/2, rc 0, transno 4294975801 00000100:00100000:1.0:1783520935.812856:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993109376:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.812867:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44307 00000100:00100000:0.0:1783520935.812875:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507938700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+236:3625:x1870152993111168:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.812887:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1543 count 1 00000100:00100000:1.0:1783520935.812894:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:0.0:1783520935.812898:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000059c8dff3/0xd5401cbabc02576e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x607:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.812902:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350793b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993111296:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.812914:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000071dd8c77/0xd5401cbabc01fb5d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x607:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0628 expref: 765 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.812919:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x607:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.812922:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x607:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.812925:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000059c8dff3/0xd5401cbabc02576e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x607:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.812930:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.812948:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000071dd8c77/0xd5401cbabc01fb5d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x607:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0628 expref: 765 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.812956:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000071dd8c77/0xd5401cbabc01fb5d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x607:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0628 expref: 765 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.812965:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530f480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993111424:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.812966:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993111296 00000100:00100000:1.0:1783520935.812969:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993111296 00000100:00100000:0.0:1783520935.812993:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470d80 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.812994:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44308 00000100:00100000:1.0:1783520935.813003:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350793ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+237:3625:x1870152993111296:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.813013:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1544 count 1 00000100:00100000:0.0:1783520935.813021:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993110528:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.813023:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025775 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x608:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.813030:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000003680c3ad/0xd5401cbabc025752 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x603:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.813035:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008725390a/0xd5401cbabc01fb64 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x608:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0636 expref: 765 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.813044:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x608:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.813046:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x608:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.813049:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025775 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x608:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.813062:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008725390a/0xd5401cbabc01fb64 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x608:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0636 expref: 765 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.813069:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008725390a/0xd5401cbabc01fb64 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x608:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0636 expref: 765 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.813077:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350793a300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993111552:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.813112:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6300 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.813137:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5baa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993110784:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.813154:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507acea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993110016:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.813157:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025759 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x605:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.813164:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.813177:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc02574b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x602:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.813186:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002393472 00000100:00100000:2.0:1783520935.813191:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002393472 00010000:00010000:2.0:1783520935.813210:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005310f324/0xd5401cbabc01fb3a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x602:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05e2 expref: 766 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.813211:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530e300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993110912:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.813223:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000004b9eec72/0xd5401cbabc025760 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x604:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.813257:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35893 00000100:00100000:2.0:1783520935.813266:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+766:2388:x1870153002393472:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.813277:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.813283:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005310f324/0xd5401cbabc01fb3a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x602:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05e2 expref: 766 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.813294:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005310f324/0xd5401cbabc01fb3a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x602:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05e2 expref: 766 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.813308:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005310f324/0xd5401cbabc01fb3a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x602:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05e2 expref: 766 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.813319:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x602:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.813322:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x602:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.813326:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc02574b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x602:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.813334:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc02574b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x602:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.813354:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.813384:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc02574b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x602:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.813391:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc02574b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x602:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.813396:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc02574b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x602:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.813400:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc02574b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x602:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.813412:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+765:2388:x1870153002393472:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 145us (250us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.813419:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520935.813424:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35893 00010000:00010000:0.0:1783520935.813427:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.813433:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc02574b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x602:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.813441:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa7400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.813446:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc02574b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x602:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.813455:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc02574b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x602:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.813469:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc02574b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x602:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.813475:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006847d995/0xd5401cbabc02574b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x602:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.813491:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.813493:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.813507:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.813516:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507acd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+238:3627:x1870152993109760:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2545us (3110us total) trans 4294975802 rc 0/0 00000100:00100000:1.0:1783520935.813522:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44302 00000100:00100000:0.0:1783520935.813544:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000004:00080000:1.0:1783520935.813548:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507acce00/2, rc 0, transno 4294975802 00000100:00100000:1.0:1783520935.813555:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507acce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993109760:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.813572:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002393728 00000100:00100000:2.0:1783520935.813577:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002393728 00000100:00100000:0.0:1783520935.813580:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.813597:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000499c5659/0xd5401cbabc01fb4f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x605:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f060c expref: 765 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.813603:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1783520935.813614:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507939500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993111680:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.813622:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002393344 00000100:00100000:0.0:1783520935.813627:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.813628:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002393344 00010000:00010000:2.0:1783520935.813647:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000033ca9bdc/0xd5401cbabc01fb2c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x601:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05d4 expref: 766 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.813659:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.813661:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.813674:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002393600 00000100:00100000:2.0:1783520935.813678:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002393600 00010000:00010000:2.0:1783520935.813695:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ee13ff38/0xd5401cbabc01fb41 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x603:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05f0 expref: 767 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.813708:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002393856 00000100:00100000:1.0:1783520935.813715:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002393856 00000100:00100000:2.0:1783520935.813719:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35894 00000100:00100000:2.0:1783520935.813729:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530e300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+768:2388:x1870153002393728:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.813738:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.813739:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000817fcacc/0xd5401cbabc01fb48 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x604:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05fe expref: 768 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.813746:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000499c5659/0xd5401cbabc01fb4f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x605:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f060c expref: 768 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.813763:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000499c5659/0xd5401cbabc01fb4f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x605:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f060c expref: 768 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.813767:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.813771:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35895 00000100:00100000:1.0:1783520935.813786:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+768:2387:x1870153002393344:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.813789:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000499c5659/0xd5401cbabc01fb4f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x605:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f060c expref: 768 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.813796:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.813806:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000033ca9bdc/0xd5401cbabc01fb2c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x601:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05d4 expref: 768 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.813810:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x605:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.813818:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x605:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.813824:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025759 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x605:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.813826:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000033ca9bdc/0xd5401cbabc01fb2c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x601:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05d4 expref: 767 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.813844:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025759 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x605:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.813852:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000033ca9bdc/0xd5401cbabc01fb2c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x601:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05d4 expref: 767 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.813862:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x601:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.813865:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x601:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.813868:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.813868:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000013dcb818/0xd5401cbabc025744 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x601:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.813876:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000013dcb818/0xd5401cbabc025744 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x601:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.813899:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.813901:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530e300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+766:2388:x1870153002393728:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 173us (357us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.813910:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35894 00010000:00010000:0.0:1783520935.813916:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000013dcb818/0xd5401cbabc025744 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x601:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.813923:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000013dcb818/0xd5401cbabc025744 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x601:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.813927:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002393984 00000100:00100000:1.0:1783520935.813928:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+765:2387:x1870153002393344:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 143us (346us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.813933:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002393984 00010000:00010000:0.0:1783520935.813935:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000013dcb818/0xd5401cbabc025744 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x601:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.813942:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35895 00010000:00010000:0.0:1783520935.813944:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000013dcb818/0xd5401cbabc025744 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x601:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.813953:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000fd59ddf1/0xd5401cbabc022977 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x606:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3067 expref: 766 pid: 9151 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.813961:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.813968:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.813975:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000013dcb818/0xd5401cbabc025744 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x601:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.813978:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35896 00010000:00010000:1.0:1783520935.813989:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf7e00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520935.813991:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638585c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+765:2387:x1870153002393600:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.813995:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000013dcb818/0xd5401cbabc025744 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x601:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.814001:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.814005:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000013dcb818/0xd5401cbabc025744 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x601:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.814009:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ee13ff38/0xd5401cbabc01fb41 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x603:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05f0 expref: 765 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.814026:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000013dcb818/0xd5401cbabc025744 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x601:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.814027:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ee13ff38/0xd5401cbabc01fb41 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x603:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05f0 expref: 765 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.814045:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000013dcb818/0xd5401cbabc025744 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x601:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.814052:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507acc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+238:3627:x1870152993109632:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3347us (3747us total) trans 4294975803 rc 0/0 00010000:00010000:2.0:1783520935.814057:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ee13ff38/0xd5401cbabc01fb41 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x603:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05f0 expref: 765 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.814064:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44301 00010000:00010000:1.0:1783520935.814065:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.814068:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.814073:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.814077:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x603:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.814082:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x603:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.814087:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000003680c3ad/0xd5401cbabc025752 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x603:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.814090:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993111680 00000004:00080000:1.0:1783520935.814094:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ace300/2, rc 0, transno 4294975803 00000100:00100000:0.0:1783520935.814095:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993111680 00010000:00010000:2.0:1783520935.814099:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003680c3ad/0xd5401cbabc025752 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x603:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.814104:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ace300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993109632:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.814129:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.814129:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44309 00000100:00100000:0.0:1783520935.814142:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514fdc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+238:3628:x1870152993111680:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.814149:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514fdce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993111808:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.814157:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638585c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+764:2387:x1870153002393600:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 167us (530us total) trans 0 rc 0/0 00002000:00080000:0.0:1783520935.814158:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1545 count 1 00000100:00100000:2.0:1783520935.814169:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35896 00000100:00100000:1.0:1783520935.814176:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.814176:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003f15af42/0xd5401cbabc02577c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x609:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.814191:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000000e63e41f/0xd5401cbabc01fb6b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x609:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0644 expref: 763 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.814205:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x609:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.814208:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x609:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.814212:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003f15af42/0xd5401cbabc02577c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x609:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.814219:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35897 00000100:00100000:1.0:1783520935.814230:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35898 00000100:00100000:2.0:1783520935.814235:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530c000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+763:2387:x1870153002393856:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.814247:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.814247:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530d500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+763:2387:x1870153002393984:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.814247:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000000e63e41f/0xd5401cbabc01fb6b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x609:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0644 expref: 763 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.814257:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000817fcacc/0xd5401cbabc01fb48 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x604:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05fe expref: 763 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.814261:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.814268:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000000e63e41f/0xd5401cbabc01fb6b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x609:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0644 expref: 763 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.814270:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000fd59ddf1/0xd5401cbabc022977 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x606:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3067 expref: 763 pid: 9151 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.814284:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000817fcacc/0xd5401cbabc01fb48 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x604:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05fe expref: 763 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.814295:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000fd59ddf1/0xd5401cbabc022977 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x606:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3067 expref: 763 pid: 9151 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.814301:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530e680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993111936:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.814318:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x606:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.814322:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x606:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.814327:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000817fcacc/0xd5401cbabc01fb48 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x604:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f05fe expref: 763 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.814327:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025767 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x606:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.814345:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025767 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x606:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.814353:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x604:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.814354:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470b40 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.814356:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.814384:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x604:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.814389:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000004b9eec72/0xd5401cbabc025760 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x604:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.814393:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530d500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+762:2387:x1870153002393984:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 153us (632us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.814405:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35898 00010000:00010000:2.0:1783520935.814407:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000004b9eec72/0xd5401cbabc025760 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x604:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.814429:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993111808 00010000:00010000:2.0:1783520935.814431:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.814434:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993111808 00000100:00100000:2.0:1783520935.814456:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530c000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+761:2387:x1870153002393856:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 225us (797us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.814468:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35897 00000100:00100000:1.0:1783520935.814468:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44310 00000100:00100000:1.0:1783520935.814481:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514fded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+239:3628:x1870152993111808:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.814486:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000004b9eec72/0xd5401cbabc025760 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x604:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.814493:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1546 count 1 00010000:00010000:0.0:1783520935.814509:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000004b9eec72/0xd5401cbabc025760 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x604:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.814518:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ea31747f/0xd5401cbabc025783 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x60a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.814520:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000004b9eec72/0xd5401cbabc025760 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x604:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.814532:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007eb34d50/0xd5401cbabc01fb72 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x60a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0652 expref: 760 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.814537:0:8408:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000fd59ddf1/0xd5401cbabc022977 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x606:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3067 expref: 760 pid: 9151 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.814537:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000004b9eec72/0xd5401cbabc025760 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x604:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.814539:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x60a:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.814547:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x60a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.814554:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ea31747f/0xd5401cbabc025783 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x60a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.814570:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5bb480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993111040:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.814591:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007eb34d50/0xd5401cbabc01fb72 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x60a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0652 expref: 759 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.814593:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025767 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x606:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.814595:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.814617:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007eb34d50/0xd5401cbabc01fb72 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x60a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0652 expref: 759 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.814620:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.814628:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000004b9eec72/0xd5401cbabc025760 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x604:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.814634:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514fdf480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993112064:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.814644:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf7200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.814648:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000004b9eec72/0xd5401cbabc025760 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x604:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.814657:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000004b9eec72/0xd5401cbabc025760 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x604:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.814671:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000004b9eec72/0xd5401cbabc025760 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x604:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.814675:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6cc0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.814679:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004b9eec72/0xd5401cbabc025760 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x604:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.814698:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.814701:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.814705:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.814711:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507939180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+240:3626:x1870152993110272:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2680us (2810us total) trans 4294975804 rc 0/0 00000004:00080000:1.0:1783520935.814718:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ae5180/2, rc 0, transno 4294975804 00010000:00010000:2.0:1783520935.814719:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025767 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x606:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.814723:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44305 00000100:00100000:1.0:1783520935.814730:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ae5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993110272:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.814737:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025767 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x606:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.814747:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025767 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x606:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.814761:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000003680c3ad/0xd5401cbabc025752 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x603:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.814763:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025767 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x606:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.814767:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1783520935.814777:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514fdfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993112192:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.814784:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000003680c3ad/0xd5401cbabc025752 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x603:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.814794:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.814795:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003680c3ad/0xd5401cbabc025752 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x603:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.814806:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000003680c3ad/0xd5401cbabc025752 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x603:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.814807:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520935.814814:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.814815:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025767 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x606:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.814830:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.814834:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025767 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x606:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.814837:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.814846:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025767 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x606:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.814851:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.814858:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000003680c3ad/0xd5401cbabc025752 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x603:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.814859:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025767 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x606:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.814876:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf7a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.814878:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025767 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x606:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.814883:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000003680c3ad/0xd5401cbabc025752 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x603:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.814900:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000003680c3ad/0xd5401cbabc025752 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x603:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.814903:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.814907:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.814911:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.814916:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000003680c3ad/0xd5401cbabc025752 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x603:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.814924:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003680c3ad/0xd5401cbabc025752 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x603:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.814942:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.814945:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.814947:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.814959:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993112192 00000100:00100000:0.0:1783520935.814964:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993112192 00000100:00100000:0.0:1783520935.814999:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44311 00000100:00100000:2.0:1783520935.815012:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507938380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+241:3628:x1870152993110656:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2802us (2971us total) trans 4294975805 rc 0/0 00000100:00100000:0.0:1783520935.815013:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514fdf100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+241:3626:x1870152993112192:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.815026:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44306 00002000:00080000:0.0:1783520935.815031:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1547 count 1 00010000:00010000:0.0:1783520935.815045:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc02578a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x60b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.815057:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000069fb8b6b/0xd5401cbabc01fb79 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x60b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0660 expref: 759 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520935.815064:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507939f80/2, rc 0, transno 4294975805 00010000:00010000:0.0:1783520935.815064:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x60b:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.815072:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x60b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.815077:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc02578a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x60b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.815081:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507939f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993110656:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.815109:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000069fb8b6b/0xd5401cbabc01fb79 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x60b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0660 expref: 759 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.815116:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514fde300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993112320:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.815124:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000069fb8b6b/0xd5401cbabc01fb79 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x60b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0660 expref: 759 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.815138:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530ca80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993112448:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.815148:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.815176:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470780 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.815207:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530f480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993111424:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.815226:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000059c8dff3/0xd5401cbabc02576e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x607:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.815228:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350793a300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993111552:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.815245:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.815254:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025775 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x608:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.815278:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002394112 00000100:00100000:1.0:1783520935.815282:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002394112 00010000:00010000:1.0:1783520935.815303:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000071dd8c77/0xd5401cbabc01fb5d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x607:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0628 expref: 760 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.815330:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35899 00000100:00100000:0.0:1783520935.815335:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ae4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+241:3625:x1870152993110144:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3673us (3741us total) trans 4294975806 rc 0/0 00000100:00100000:2.0:1783520935.815342:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.815342:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5bb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+760:2387:x1870153002394112:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.815351:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44303 00010000:00010000:1.0:1783520935.815355:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.815377:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000071dd8c77/0xd5401cbabc01fb5d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x607:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0628 expref: 760 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.815387:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000071dd8c77/0xd5401cbabc01fb5d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x607:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0628 expref: 760 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.815394:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993112320 00000100:00100000:0.0:1783520935.815399:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993112320 00010000:00010000:1.0:1783520935.815400:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000071dd8c77/0xd5401cbabc01fb5d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x607:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0628 expref: 760 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.815414:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x607:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.815416:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x607:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.815420:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000059c8dff3/0xd5401cbabc02576e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x607:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.815429:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44312 00010000:00010000:1.0:1783520935.815430:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000059c8dff3/0xd5401cbabc02576e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x607:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.815439:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514fdd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+241:3626:x1870152993112320:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.815454:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00002000:00080000:0.0:1783520935.815454:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1548 count 1 00010000:00010000:0.0:1783520935.815470:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc025791 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x60c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.815487:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025759 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x605:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.815488:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5bb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+759:2387:x1870153002394112:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 149us (243us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.815488:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000736f3890/0xd5401cbabc01fb80 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x60c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f066e expref: 759 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.815515:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35899 00010000:00010000:0.0:1783520935.815527:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x60c:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.815528:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025759 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x605:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.815530:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002394240 00010000:00010000:0.0:1783520935.815533:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x60c:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.815535:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002394240 00010000:00010000:0.0:1783520935.815543:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc025791 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x60c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.815547:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025759 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x605:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.815548:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008725390a/0xd5401cbabc01fb64 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x608:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0636 expref: 759 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.815573:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025759 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x605:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.815580:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000736f3890/0xd5401cbabc01fb80 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x60c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f066e expref: 759 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.815585:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35900 00010000:00010000:0.0:1783520935.815598:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000736f3890/0xd5401cbabc01fb80 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x60c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f066e expref: 759 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.815599:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.815601:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5b9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+759:2389:x1870153002394240:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.815621:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520935.815621:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530f480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993112576:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.815629:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025759 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x605:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.815638:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.815643:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025759 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x605:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.815649:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025759 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x605:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.815662:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025759 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x605:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.815664:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470d80 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.815673:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025759 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x605:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.815677:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000059c8dff3/0xd5401cbabc02576e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x607:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.815688:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000059c8dff3/0xd5401cbabc02576e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x607:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.815697:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.815697:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000059c8dff3/0xd5401cbabc02576e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x607:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.815702:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.815704:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350793aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+242:3628:x1870152993110400:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3712us (3890us total) trans 4294975807 rc 0/0 00010000:00010000:1.0:1783520935.815706:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.815722:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44304 00010000:00010000:0.0:1783520935.815724:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000059c8dff3/0xd5401cbabc02576e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x607:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.815735:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507acc000/2, rc 0, transno 4294975806 00010000:00010000:0.0:1783520935.815741:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.815746:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507acc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993110144:0@lo:6:osp-syn-1-0.0 00000004:00080000:2.0:1783520935.815765:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350793b800/2, rc 0, transno 4294975807 00000100:00100000:2.0:1783520935.815775:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350793b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993110400:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.815782:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1783520935.815793:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363fe54e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993112832:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.815804:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.815810:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008725390a/0xd5401cbabc01fb64 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x608:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0636 expref: 759 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.815823:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008725390a/0xd5401cbabc01fb64 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x608:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0636 expref: 759 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.815828:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507938700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+242:3625:x1870152993111168:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2952us (3030us total) trans 4294975808 rc 0/0 00000100:00100000:2.0:1783520935.815834:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.815846:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44307 00010000:00010000:1.0:1783520935.815853:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008725390a/0xd5401cbabc01fb64 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x608:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0636 expref: 759 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.815863:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993112832 00000100:00100000:2.0:1783520935.815866:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350793a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993112704:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.815868:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993112832 00010000:00010000:1.0:1783520935.815873:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x608:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.815878:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x608:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.815883:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025775 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x608:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.815889:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025775 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x608:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.815900:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44313 00000100:00100000:2.0:1783520935.815905:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.815907:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.815914:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5b9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+242:3628:x1870152993112832:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.815928:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1550 count 1 00000100:00100000:1.0:1783520935.815931:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5b9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+758:2389:x1870153002394240:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 336us (595us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.815938:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35900 00010000:00010000:2.0:1783520935.815943:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.815943:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000168629d0/0xd5401cbabc025798 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x60e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.815951:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000059c8dff3/0xd5401cbabc02576e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x607:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.815962:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf6800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.815962:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004c56bf07/0xd5401cbabc022866 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x60e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f56 expref: 757 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.815968:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000059c8dff3/0xd5401cbabc02576e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x607:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.815974:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507938e00/2, rc 0, transno 4294975808 00010000:00010000:0.0:1783520935.815974:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x60e:0x0].0x0 since=36s v=(0 0) 00010000:00010000:0.0:1783520935.815983:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x60e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.815984:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000059c8dff3/0xd5401cbabc02576e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x607:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.815988:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000168629d0/0xd5401cbabc025798 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x60e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.815990:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507938e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993111168:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.816009:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000059c8dff3/0xd5401cbabc02576e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x607:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.816018:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000059c8dff3/0xd5401cbabc02576e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x607:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.816024:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004c56bf07/0xd5401cbabc022866 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x60e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f56 expref: 757 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.816041:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:2.0:1783520935.816044:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.816044:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004c56bf07/0xd5401cbabc022866 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x60e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f56 expref: 757 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.816049:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.816051:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363fe55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993113088:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.816054:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.816059:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993112960:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.816072:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993112704 00000100:00100000:2.0:1783520935.816077:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993112704 00000100:00100000:1.0:1783520935.816083:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.816090:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470f00 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.816104:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993113088 00000100:00100000:2.0:1783520935.816108:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993113088 00000100:00100000:0.0:1783520935.816113:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44314 00010000:00010000:1.0:1783520935.816122:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025775 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x608:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.816123:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5baa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+244:3628:x1870152993112704:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.816132:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44315 00002000:00080000:0.0:1783520935.816139:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1549 count 1 00010000:00010000:1.0:1783520935.816141:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025775 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x608:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.816144:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363fe55c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+244:3626:x1870152993113088:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.816152:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000c90bcf0/0xd5401cbabc02579f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x60d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.816159:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025775 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x608:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.816160:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1551 count 1 00010000:00010000:0.0:1783520935.816164:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000783c9f87/0xd5401cbabc01fb87 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x60d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f067c expref: 757 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.816174:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0257a6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x60f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.816185:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x60d:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.816187:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025775 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x608:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.816192:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x60d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.816197:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009f09c2eb/0xd5401cbabc01fb95 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x60f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0698 expref: 757 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.816199:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000c90bcf0/0xd5401cbabc02579f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x60d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.816221:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.816223:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x60f:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.816229:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x60f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.816233:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0257a6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x60f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.816235:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000783c9f87/0xd5401cbabc01fb87 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x60d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f067c expref: 757 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.816256:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.816257:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000783c9f87/0xd5401cbabc01fb87 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x60d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f067c expref: 757 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.816261:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025775 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x608:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.816283:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943514fa6400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.816287:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025775 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x608:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.816287:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530e300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993113216:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.816298:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025775 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x608:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.816305:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350793ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+245:3625:x1870152993111296:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3299us (3374us total) trans 4294975809 rc 0/0 00010000:00010000:2.0:1783520935.816310:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025775 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x608:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.816316:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44308 00000100:00100000:0.0:1783520935.816317:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124706c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.816326:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007b280ca1/0xd5401cbabc025775 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x608:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.816344:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.816346:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.816349:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.816382:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000004:00080000:1.0:1783520935.816391:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350793b480/2, rc 0, transno 4294975809 00010000:00010000:2.0:1783520935.816393:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009f09c2eb/0xd5401cbabc01fb95 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x60f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0698 expref: 757 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.816403:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350793b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993111296:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.816405:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009f09c2eb/0xd5401cbabc01fb95 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x60f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0698 expref: 757 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.816418:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b8000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993113344:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.816438:0:35277:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002394496 00000100:00100000:1.0:1783520935.816443:0:35277:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002394496 00000100:00100000:2.0:1783520935.816469:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872f00 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.816483:0:35277:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98407 00000100:00100000:1.0:1783520935.816493:0:35277:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_008:25342ed3-dd17-4439-8398-d8002e9f9f99+12:251800:x1870153002394496:12345-192.168.204.33@tcp:101:checkfiemap.0 00010000:00010000:1.0:1783520935.816518:0:35277:0:(ldlm_lockd.c:1265:ldlm_handle_enqueue()) ### server-side enqueue handler START 00010000:00010000:1.0:1783520935.816531:0:35277:0:(ldlm_lockd.c:1343:ldlm_handle_enqueue()) ### server-side enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff9436104ec000/0xd5401cbabc0257ad lrc: 2/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT gid 0 flags: 0x40000000000000 nid: local remote: 0xc275876b859f34dc expref: -99 pid: 35277 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.816649:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530e680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993111936:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.816656:0:35277:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff9435122ddc00/0xd5401cbabc0257b4 lrc: 3/1,0 mode: --/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x0/0x0 rrc: 7 type: IBT flags: 0x40000000000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.816663:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000003f15af42/0xd5401cbabc02577c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x609:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.816671:0:35277:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff9435122ddc00/0xd5401cbabc0257b4 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 7 type: IBT flags: 0x50210000000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.816676:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.816686:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530ca80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993112448:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.816688:0:35277:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff9435122ddc00/0xd5401cbabc0257b4 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 7 type: IBT flags: 0x40210000000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.816703:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002394752 00000004:00200000:1.0:1783520935.816703:0:35277:0:(mdt_handler.c:909:mdt_pack_attr2body()) [0x200000402:0x72a:0x0]: returning size 262144 00010000:00010000:0.0:1783520935.816706:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc02578a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x60b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.816710:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002394752 00010000:00010000:2.0:1783520935.816731:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007eb34d50/0xd5401cbabc01fb72 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x60a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0652 expref: 758 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.816753:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35901 00000100:00100000:2.0:1783520935.816762:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5bbb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+758:2387:x1870153002394752:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.816769:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.816773:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007eb34d50/0xd5401cbabc01fb72 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x60a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0652 expref: 758 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.816781:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007eb34d50/0xd5401cbabc01fb72 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x60a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0652 expref: 758 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.816791:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x60a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.816794:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x60a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.816797:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000ea31747f/0xd5401cbabc025783 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.816802:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ea31747f/0xd5401cbabc025783 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x60a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.816814:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.816830:0:35277:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff9436104ec000/0xd5401cbabc0257ad lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x44000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34dc expref: 15 pid: 35277 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.816843:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5bbb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+758:2387:x1870153002394752:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 81us (168us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.816851:0:35277:0:(ldlm_lockd.c:1513:ldlm_handle_enqueue()) ### server-side enqueue handler, sending reply (err=0, rc=0) ns: mdt-lustre-MDT0000_UUID lock: ffff9435122ddc00/0xd5401cbabc0257b4 lrc: 3/0,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 7 type: IBT gid 0 flags: 0x40200000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34dc expref: 14 pid: 35277 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.816853:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35901 00010000:00010000:1.0:1783520935.816865:0:35277:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x72a:0x0].0x0 (ffff943515473e00) 00010000:00010000:1.0:1783520935.816868:0:35277:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 1b 00010000:00010000:1.0:1783520935.816872:0:35277:0:(ldlm_lockd.c:1592:ldlm_handle_enqueue()) ### server-side enqueue handler END (lock ffff9435122ddc00, rc 0) 00000100:00100000:0.0:1783520935.816878:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.816905:0:35277:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_008:25342ed3-dd17-4439-8398-d8002e9f9f99+14:251800:x1870153002394496:12345-192.168.204.33@tcp:101:checkfiemap.0 Request processed in 413us (523us total) trans 8589957896 rc 0/0 00000100:00100000:1.0:1783520935.816914:0:35277:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98407 00000100:00100000:0.0:1783520935.816969:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.816986:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363fe56680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993113472:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.817019:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.817043:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993113472 00000100:00100000:0.0:1783520935.817048:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993113472 00000100:00100000:1.0:1783520935.817062:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002394624 00000100:00100000:1.0:1783520935.817068:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002394624 00000100:00100000:0.0:1783520935.817081:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44316 00010000:00010000:1.0:1783520935.817090:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000000e63e41f/0xd5401cbabc01fb6b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x609:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0644 expref: 758 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.817093:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363fe56d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+245:3626:x1870152993113472:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.817114:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1552 count 1 00000100:00100000:1.0:1783520935.817122:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002394880 00000100:00100000:1.0:1783520935.817127:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002394880 00010000:00010000:0.0:1783520935.817128:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0257c2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x610:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.817138:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b3325716/0xd5401cbabc01fb9c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x610:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f06a6 expref: 759 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.817148:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000069fb8b6b/0xd5401cbabc01fb79 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x60b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0660 expref: 759 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.817148:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x610:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.817156:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x610:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.817163:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0257c2 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x610:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.817184:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b3325716/0xd5401cbabc01fb9c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x610:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06a6 expref: 759 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.817185:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35902 00010000:00010000:0.0:1783520935.817197:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b3325716/0xd5401cbabc01fb9c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x610:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06a6 expref: 759 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.817205:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530ca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+759:2387:x1870153002394624:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.817210:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530e680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993113600:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.817211:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35903 00010000:00010000:1.0:1783520935.817220:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520935.817228:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638585450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+759:2390:x1870153002394880:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.817229:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000000e63e41f/0xd5401cbabc01fb6b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x609:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0644 expref: 759 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.817243:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.817248:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000000e63e41f/0xd5401cbabc01fb6b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x609:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0644 expref: 759 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.817250:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470780 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.817252:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000069fb8b6b/0xd5401cbabc01fb79 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x60b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0660 expref: 759 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.817276:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000069fb8b6b/0xd5401cbabc01fb79 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x60b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0660 expref: 759 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.817283:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000e63e41f/0xd5401cbabc01fb6b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x609:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0644 expref: 759 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.817301:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000069fb8b6b/0xd5401cbabc01fb79 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x60b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0660 expref: 758 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.817303:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x609:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.817311:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x609:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.817317:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000003f15af42/0xd5401cbabc02577c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x609:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.817319:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x60b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.817327:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x60b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.817330:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003f15af42/0xd5401cbabc02577c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x609:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.817332:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc02578a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.817351:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc02578a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x60b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.817383:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.817388:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.817393:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc02578a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.817403:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc02578a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.817412:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc02578a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.817413:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530ca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+757:2387:x1870153002394624:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 215us (538us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.817416:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638585450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+757:2390:x1870153002394880:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 191us (445us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.817432:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35902 00010000:00010000:0.0:1783520935.817436:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc02578a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x60b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.817445:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35903 00010000:00010000:0.0:1783520935.817465:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.817493:0:12734:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007eb34d50/0xd5401cbabc01fb72 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x60a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0652 expref: 755 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.817523:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514fdf480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993112064:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.817543:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000ea31747f/0xd5401cbabc025783 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.817552:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000ea31747f/0xd5401cbabc025783 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.817560:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000ea31747f/0xd5401cbabc025783 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.817564:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514fdf100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+246:3626:x1870152993112192:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2556us (2751us total) trans 4294975810 rc 0/0 00010000:00010000:1.0:1783520935.817573:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ea31747f/0xd5401cbabc025783 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.817578:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44311 00010000:00010000:1.0:1783520935.817587:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000ea31747f/0xd5401cbabc025783 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x60a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.817603:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:0.0:1783520935.817605:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520935.817610:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc02578a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x60b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.817614:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.817616:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf6e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.817622:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000ea31747f/0xd5401cbabc025783 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x60a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.817622:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc02578a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x60b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.817643:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc02578a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x60b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.817644:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9436104ed800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.817651:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000ea31747f/0xd5401cbabc025783 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x60a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.817657:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc02578a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x60b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.817661:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000ea31747f/0xd5401cbabc025783 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x60a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.817666:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c3ffd6bd/0xd5401cbabc02578a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x60b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.817680:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000ea31747f/0xd5401cbabc025783 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x60a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.817689:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ea31747f/0xd5401cbabc025783 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x60a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.817694:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.817698:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.817702:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.817714:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.817716:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000003f15af42/0xd5401cbabc02577c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x609:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.817717:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.817720:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.817728:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000003f15af42/0xd5401cbabc02577c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x609:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.817738:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003f15af42/0xd5401cbabc02577c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x609:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.817743:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514fdfb80/2, rc 0, transno 4294975810 00010000:00010000:0.0:1783520935.817750:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000003f15af42/0xd5401cbabc02577c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x609:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.817756:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514fdfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993112192:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.817767:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.817789:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.817795:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000003f15af42/0xd5401cbabc02577c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x609:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.817804:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf7000 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520935.817807:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363fe57100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993113728:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.817809:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000003f15af42/0xd5401cbabc02577c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x609:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.817821:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000003f15af42/0xd5401cbabc02577c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x609:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.817840:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000003f15af42/0xd5401cbabc02577c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x609:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.817848:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003f15af42/0xd5401cbabc02577c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x609:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.817857:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514fdc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+246:3628:x1870152993111680:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3715us (4198us total) trans 4294975811 rc 0/0 00000100:00100000:1.0:1783520935.817858:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.817867:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44309 00010000:00010000:2.0:1783520935.817879:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.817883:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.817887:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.817887:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993113728 00000100:00100000:0.0:1783520935.817892:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993113728 00000004:00080000:2.0:1783520935.817917:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507939500/2, rc 0, transno 4294975811 00000100:00100000:0.0:1783520935.817918:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44317 00000100:00100000:0.0:1783520935.817927:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363fe56300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+246:3627:x1870152993113728:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.817930:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507939500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993111680:0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.817941:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1553 count 1 00010000:00010000:0.0:1783520935.817954:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0257c9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x611:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.817968:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f40d9167/0xd5401cbabc01fba3 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x611:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f06b4 expref: 754 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.817976:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x611:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.817980:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x611:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.817981:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993113856:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.817986:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0257c9 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x611:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.818012:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f40d9167/0xd5401cbabc01fba3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x611:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06b4 expref: 754 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.818024:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f40d9167/0xd5401cbabc01fba3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x611:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06b4 expref: 754 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.818026:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.818027:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514fded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+247:3628:x1870152993111808:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3546us (3847us total) trans 4294975812 rc 0/0 00000100:00100000:1.0:1783520935.818045:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44310 00000100:00100000:0.0:1783520935.818049:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530c700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993113984:192.168.204.33@tcp:104: 00000004:00080000:2.0:1783520935.818065:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514fdce00/2, rc 0, transno 4294975812 00000100:00100000:1.0:1783520935.818070:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993113856 00000100:00100000:1.0:1783520935.818075:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993113856 00000100:00100000:2.0:1783520935.818077:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514fdce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993111808:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.818100:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.818111:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44318 00000100:00100000:1.0:1783520935.818121:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5b9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+247:3628:x1870152993113856:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.818138:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1554 count 1 00010000:00010000:1.0:1783520935.818153:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0257d0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x612:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.818165:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008043ba53/0xd5401cbabc01fbaa lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x612:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f06c2 expref: 754 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.818173:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530f480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993112576:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.818175:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x612:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.818181:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x612:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.818186:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0257d0 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x612:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.818192:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc025791 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x60c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.818212:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008043ba53/0xd5401cbabc01fbaa lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x612:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06c2 expref: 754 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.818224:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008043ba53/0xd5401cbabc01fbaa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x612:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06c2 expref: 754 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.818236:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363fe55180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993114240:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.818275:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57d80 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.818292:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993114112:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.818327:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.818340:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993112960:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.818353:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b8000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993113344:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.818354:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000168629d0/0xd5401cbabc025798 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x60e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.818394:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993114112 00000100:00100000:0.0:1783520935.818397:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993114112 00010000:00010000:2.0:1783520935.818400:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0257a6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x60f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.818424:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44319 00000100:00100000:0.0:1783520935.818432:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514fdf480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+248:3625:x1870152993114112:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.818446:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1555 count 1 00000100:00100000:2.0:1783520935.818458:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.818460:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0257d7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x613:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.818473:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000000c7c4c37/0xd5401cbabc01fbb1 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x613:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f06d0 expref: 754 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.818476:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002395136 00000100:00100000:2.0:1783520935.818480:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002395136 00010000:00010000:0.0:1783520935.818482:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x613:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.818485:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x613:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.818488:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0257d7 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x613:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.818512:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004c56bf07/0xd5401cbabc022866 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x60e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f56 expref: 755 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.818520:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000000c7c4c37/0xd5401cbabc01fbb1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x613:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06d0 expref: 755 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.818538:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000000c7c4c37/0xd5401cbabc01fbb1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x613:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06d0 expref: 755 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.818548:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993114368:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.818561:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35904 00000100:00100000:2.0:1783520935.818571:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5b8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+755:2387:x1870153002395136:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.818578:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.818582:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004c56bf07/0xd5401cbabc022866 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x60e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f56 expref: 755 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.818592:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004c56bf07/0xd5401cbabc022866 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x60e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f56 expref: 755 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.818600:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470f00 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.818606:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004c56bf07/0xd5401cbabc022866 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x60e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f56 expref: 755 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.818620:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x60e:0x0].0x0 since=0s v=(1 0) 00000100:00100000:1.0:1783520935.818624:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.818626:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x60e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.818632:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000168629d0/0xd5401cbabc025798 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.818633:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530e300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993113216:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.818646:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000168629d0/0xd5401cbabc025798 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x60e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.818652:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000000c90bcf0/0xd5401cbabc02579f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x60d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.818668:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.818669:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002395008 00000100:00100000:3.0:1783520935.818675:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002395008 00010000:00010000:0.0:1783520935.818676:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000168629d0/0xd5401cbabc025798 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.818692:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000168629d0/0xd5401cbabc025798 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.818695:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5b8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+755:2387:x1870153002395136:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 124us (237us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.818699:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000736f3890/0xd5401cbabc01fb80 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x60c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f066e expref: 755 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.818714:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000168629d0/0xd5401cbabc025798 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.818716:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35904 00000100:00100000:1.0:1783520935.818717:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.818727:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000168629d0/0xd5401cbabc025798 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x60e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.818738:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002395392 00000100:00100000:2.0:1783520935.818743:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002395392 00010000:00010000:0.0:1783520935.818743:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.818761:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009f09c2eb/0xd5401cbabc01fb95 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x60f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0698 expref: 755 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.818772:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.818788:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35905 00000100:00100000:2.0:1783520935.818798:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e004e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+755:2387:x1870153002395392:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.818806:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.818812:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009f09c2eb/0xd5401cbabc01fb95 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x60f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0698 expref: 755 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.818822:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009f09c2eb/0xd5401cbabc01fb95 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x60f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0698 expref: 755 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.818828:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5b9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+249:3628:x1870152993112832:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2917us (2996us total) trans 4294975813 rc 0/0 00000100:00100000:0.0:1783520935.818840:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44313 00010000:00010000:2.0:1783520935.818841:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009f09c2eb/0xd5401cbabc01fb95 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x60f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0698 expref: 755 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.818850:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.818869:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x60f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.818872:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x60f:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.818875:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002395264 00010000:00010000:2.0:1783520935.818877:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0257a6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.818881:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363fe54e00/2, rc 0, transno 4294975813 00010000:00010000:2.0:1783520935.818890:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0257a6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x60f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.818898:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.818898:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363fe54e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993112832:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.818914:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002395264 00010000:00010000:2.0:1783520935.818922:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.818929:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.818938:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000783c9f87/0xd5401cbabc01fb87 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x60d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f067c expref: 755 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.818946:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993114496:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.818959:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e004e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+755:2387:x1870153002395392:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 160us (246us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.818961:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002395520 00000100:00100000:3.0:1783520935.818967:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002395520 00000100:00100000:1.0:1783520935.818967:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002395776 00000100:00100000:2.0:1783520935.818973:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35905 00000100:00100000:1.0:1783520935.818974:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002395776 00000100:00100000:0.0:1783520935.818974:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.818999:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b3325716/0xd5401cbabc01fb9c lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x610:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06a6 expref: 755 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.819001:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35906 00000100:00100000:2.0:1783520935.819013:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 12255 00000100:00100000:3.0:1783520935.819015:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e007b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+755:2387:x1870153002395008:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.819027:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520935.819028:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e005500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:25342ed3-dd17-4439-8398-d8002e9f9f99+14:2390:x1870153002395520:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.819034:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35907 00010000:00010000:3.0:1783520935.819037:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000736f3890/0xd5401cbabc01fb80 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x60c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f066e expref: 755 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.819044:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.819050:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e007480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+755:2390:x1870153002395264:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.819055:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000736f3890/0xd5401cbabc01fb80 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x60c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f066e expref: 755 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.819062:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.819064:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff9435122ddc00/0xd5401cbabc0257b4 lrc: 0/0,0 mode: --/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 8 type: IBT gid 0 flags: 0x44a01000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34dc expref: 14 pid: 35277 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.819069:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530e680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993113600:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.819073:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000783c9f87/0xd5401cbabc01fb87 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x60d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f067c expref: 755 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.819076:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000736f3890/0xd5401cbabc01fb80 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x60c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f066e expref: 755 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.819083:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0257c2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x610:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.819091:0:17759:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x72a:0x0].0x0 (ffff943515473e00) 00010000:00010000:2.0:1783520935.819100:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.819108:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000783c9f87/0xd5401cbabc01fb87 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x60d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f067c expref: 755 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.819114:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002395648 00000100:00100000:0.0:1783520935.819119:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002395648 00010000:00010000:1.0:1783520935.819126:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000783c9f87/0xd5401cbabc01fb87 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x60d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f067c expref: 755 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.819131:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e005500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:25342ed3-dd17-4439-8398-d8002e9f9f99+13:2390:x1870153002395520:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 106us (234us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.819142:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993114496 00000100:00100000:2.0:1783520935.819148:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 12255 00000100:00100000:0.0:1783520935.819148:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993114496 00010000:00010000:1.0:1783520935.819149:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x60d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.819155:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x60d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.819160:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000000c90bcf0/0xd5401cbabc02579f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.819170:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000c90bcf0/0xd5401cbabc02579f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x60d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.819176:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35908 00000100:00100000:0.0:1783520935.819179:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 44320 00000100:00100000:2.0:1783520935.819191:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e007800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+753:2388:x1870153002395776:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.819195:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x60c:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.819196:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635601c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:25342ed3-dd17-4439-8398-d8002e9f9f99+31:251800:x1870153002395648:12345-192.168.204.33@tcp:7:checkfiemap.0 00010000:00010000:1.0:1783520935.819202:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.819204:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.819211:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b3325716/0xd5401cbabc01fb9c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x610:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06a6 expref: 753 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.819223:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b3325716/0xd5401cbabc01fb9c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x610:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06a6 expref: 753 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.819228:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e007480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+753:2390:x1870153002395264:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 182us (382us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.819237:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0257de lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.819247:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35907 00010000:00010000:2.0:1783520935.819248:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b3325716/0xd5401cbabc01fb9c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x610:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06a6 expref: 753 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.819256:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0250bb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34a4 expref: 753 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.819264:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x60c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.819274:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc025791 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.819279:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44321 00010000:00010000:0.0:1783520935.819281:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x8f7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.819285:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x610:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.819292:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+249:3626:x1870152993114496:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.819292:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f7:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.819294:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x610:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.819305:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0257de lrc: 4/1,0 mode: --/PR res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 3 type: EXT [0->266239] (req 0->266239) gid 0 flags: 0x50010000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.819306:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0257c2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x610:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.819307:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc025791 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x60c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.819313:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1556 count 1 00010000:00010000:1.0:1783520935.819328:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0257e5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x614:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.819332:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0257c2 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x610:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.819351:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.819352:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0250bb lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000000020 nid: 192.168.204.33@tcp remote: 0xc275876b859f34a4 expref: 751 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.819385:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000df0b1cb8/0xd5401cbabc01fbb8 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x614:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f06de expref: 751 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.819390:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0250bb lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400000020 nid: 192.168.204.33@tcp remote: 0xc275876b859f34a4 expref: 751 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.819391:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.819401:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x614:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.819409:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x614:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.819413:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530e680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993114624:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.819415:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0257e5 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x614:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.819421:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e007800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+751:2388:x1870153002395776:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 230us (491us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.819435:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35908 00010000:00010000:1.0:1783520935.819444:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000df0b1cb8/0xd5401cbabc01fbb8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x614:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06de expref: 750 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.819451:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000df0b1cb8/0xd5401cbabc01fbb8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x614:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06de expref: 750 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.819455:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.819463:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000168629d0/0xd5401cbabc025798 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x60e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.819465:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513138000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993114752:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.819478:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf6c00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:3.0:1783520935.819479:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e007b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+750:2387:x1870153002395008:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 466us (858us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.819484:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000168629d0/0xd5401cbabc025798 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x60e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.819489:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470780 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.819492:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35906 00010000:00010000:2.0:1783520935.819513:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000168629d0/0xd5401cbabc025798 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x60e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.819515:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6cc0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.819518:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000000c90bcf0/0xd5401cbabc02579f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.819534:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000000c90bcf0/0xd5401cbabc02579f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.819537:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000168629d0/0xd5401cbabc025798 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x60e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.819538:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc025791 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.819554:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000c90bcf0/0xd5401cbabc02579f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.819563:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000168629d0/0xd5401cbabc025798 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x60e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.819569:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000000c90bcf0/0xd5401cbabc02579f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x60d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.819589:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.819590:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.819596:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000000c90bcf0/0xd5401cbabc02579f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x60d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.819600:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.819606:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeae00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.819613:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000000c90bcf0/0xd5401cbabc02579f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x60d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.819616:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc025791 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.819620:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530c700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993113984:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.819636:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000000c90bcf0/0xd5401cbabc02579f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x60d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.819637:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0257c9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x611:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.819639:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.819641:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc025791 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.819664:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc025791 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x60c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.819670:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000000c90bcf0/0xd5401cbabc02579f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x60d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.819680:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000c90bcf0/0xd5401cbabc02579f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x60d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.819685:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.819685:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002395904 00000100:00100000:1.0:1783520935.819692:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002395904 00010000:00010000:2.0:1783520935.819700:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.819704:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc025791 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x60c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.819705:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.819711:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf6600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.819715:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc025791 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x60c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.819720:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc025791 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x60c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.819728:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc025791 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x60c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.819728:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008043ba53/0xd5401cbabc01fbaa lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x612:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06c2 expref: 750 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.819748:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006e15f1d7/0xd5401cbabc025791 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x60c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.819766:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35909 00010000:00010000:2.0:1783520935.819767:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.819771:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.819774:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.819778:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638587050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+750:2388:x1870153002395904:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520935.819787:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514fdd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+251:3626:x1870152993112320:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4348us (4636us total) trans 4294975814 rc 0/0 00010000:00010000:1.0:1783520935.819789:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.819797:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008043ba53/0xd5401cbabc01fbaa lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x612:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06c2 expref: 750 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.819802:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0257a6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.819810:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008043ba53/0xd5401cbabc01fbaa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x612:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06c2 expref: 750 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.819821:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0257a6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.819830:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5baa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+251:3628:x1870152993112704:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3708us (3926us total) trans 4294975815 rc 0/0 00010000:00010000:1.0:1783520935.819831:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x612:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.819835:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0257a6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x60f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.819841:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x612:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.819849:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44314 00010000:00010000:1.0:1783520935.819853:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0257d0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x612:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.819858:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44312 00010000:00010000:2.0:1783520935.819858:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0257a6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x60f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.819873:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0257d0 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x612:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.819885:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350793a300/2, rc 0, transno 4294975815 00010000:00010000:2.0:1783520935.819889:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.819893:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.819899:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520935.819900:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350793a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993112704:0@lo:6:osp-syn-1-0.0 00000004:00080000:3.0:1783520935.819905:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514fde300/2, rc 0, transno 4294975814 00010000:00010000:2.0:1783520935.819907:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0257a6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x60f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.819913:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514fde300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993112320:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.819920:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638587050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+750:2388:x1870153002395904:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 144us (283us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.819923:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184c00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520935.819927:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35909 00010000:00010000:2.0:1783520935.819929:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0257a6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x60f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.819940:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0257a6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x60f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.819955:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0257a6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x60f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.819964:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0257a6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x60f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.819978:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993114880:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.819981:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.819984:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.819988:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.819988:0:12734:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008043ba53/0xd5401cbabc01fbaa lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x612:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06c2 expref: 749 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.820003:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363fe55180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993114240:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.820015:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.820020:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0257d0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x612:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.820032:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0257d0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x612:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.820039:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0257d0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x612:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.820043:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:1.0:1783520935.820052:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0257d0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x612:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.820056:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993115008:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.820067:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0257d0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x612:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.820084:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520935.820093:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363fe55c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+250:3626:x1870152993113088:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3951us (4010us total) trans 4294975816 rc 0/0 00000100:00100000:0.0:1783520935.820094:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:3.0:1783520935.820097:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363fe55500/2, rc 0, transno 4294975816 00000100:00100000:2.0:1783520935.820103:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44315 00000100:00100000:3.0:1783520935.820110:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363fe55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993113088:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.820120:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993114880 00010000:00010000:0.0:1783520935.820123:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:2.0:1783520935.820124:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993114880 00010000:00010000:0.0:1783520935.820129:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0257d0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x612:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.820136:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9435122dc400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.820139:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0257d0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x612:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.820146:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0257d0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x612:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.820147:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44322 00000100:00100000:3.0:1783520935.820153:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1783520935.820158:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+251:3627:x1870152993114880:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.820160:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0257d0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x612:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.820163:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c552d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993115136:0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.820169:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1557 count 1 00000100:00100000:1.0:1783520935.820176:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5b9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+251:3628:x1870152993113856:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2053us (2149us total) trans 4294975817 rc 0/0 00010000:00010000:0.0:1783520935.820177:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0257d0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x612:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.820183:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0257ec lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x615:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.820195:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44318 00000100:00100000:3.0:1783520935.820202:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.820205:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.820209:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.820211:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000999101c4/0xd5401cbabc01fbbf lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x615:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f06ec expref: 748 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.820212:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.820219:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993115008 00010000:00010000:2.0:1783520935.820221:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x615:0x0].0x0 since=37s v=(0 0) 00000100:00100000:1.0:1783520935.820224:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993115008 00010000:00010000:2.0:1783520935.820226:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x615:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.820232:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0257ec lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x615:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.820236:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993115136 00000004:00080000:0.0:1783520935.820237:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c5bb480/2, rc 0, transno 4294975817 00000100:00100000:3.0:1783520935.820242:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993115136 00000100:00100000:0.0:1783520935.820251:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993113856:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.820253:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44323 00010000:00010000:2.0:1783520935.820261:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000999101c4/0xd5401cbabc01fbbf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x615:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06ec expref: 748 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.820269:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+252:3627:x1870152993115008:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.820273:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44324 00010000:00010000:0.0:1783520935.820273:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0257c2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x610:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.820277:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000999101c4/0xd5401cbabc01fbbf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x615:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06ec expref: 748 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.820286:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c550000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+252:3626:x1870152993115136:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.820289:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1558 count 1 00010000:00010000:0.0:1783520935.820292:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0257c2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x610:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520935.820303:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1559 count 1 00010000:00010000:1.0:1783520935.820310:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc0257f3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x616:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.820310:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0257c2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x610:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.820311:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b8380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993115264:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.820316:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0257fa lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x617:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.820336:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0257c2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x610:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.820346:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872f00 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.820346:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000025a91e3c/0xd5401cbabc022ac0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x616:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f31b0 expref: 748 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.820347:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000048b513d5/0xd5401cbabc01fbcd lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x617:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0708 expref: 748 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.820421:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x616:0x0].0x0 since=36s v=(0 0) 00010000:00010000:1.0:1783520935.820425:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x616:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.820430:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc0257f3 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x616:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.820445:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.820445:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.820453:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0257c2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x610:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.820462:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.820466:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0257c2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x610:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.820475:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0257c2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x610:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.820491:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0257c2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x610:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.820509:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0257c2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x610:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.820529:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.820532:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.820536:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x617:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.820536:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.820561:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.820580:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x617:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.820585:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0257fa lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x617:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.820589:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002396032 00000100:00100000:2.0:1783520935.820596:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002396032 00010000:00010000:3.0:1783520935.820614:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000048b513d5/0xd5401cbabc01fbcd lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x617:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0708 expref: 749 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.820621:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363fe56d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+253:3626:x1870152993113472:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3529us (3598us total) trans 4294975818 rc 0/0 00010000:00010000:2.0:1783520935.820622:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f40d9167/0xd5401cbabc01fba3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x611:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06b4 expref: 749 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.820627:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000048b513d5/0xd5401cbabc01fbcd lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x617:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0708 expref: 749 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.820633:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.820638:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44316 00000100:00100000:3.0:1783520935.820668:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c553100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993115392:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.820680:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993114368:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.820686:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002396160 00000100:00100000:2.0:1783520935.820691:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002396160 00010000:00010000:0.0:1783520935.820698:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0257d7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x613:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.820709:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000000c7c4c37/0xd5401cbabc01fbb1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x613:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06d0 expref: 750 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.820718:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967840 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.820739:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35910 00000100:00100000:0.0:1783520935.820743:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530e680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993114624:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.820750:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513139500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+750:2389:x1870153002396032:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000004:00080000:3.0:1783520935.820754:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363fe56680/2, rc 0, transno 4294975818 00010000:00010000:0.0:1783520935.820759:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0257de lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 3 type: EXT [0->266239] (req 0->266239) gid 0 flags: 0x40010000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.820760:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.820770:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f40d9167/0xd5401cbabc01fba3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x611:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06b4 expref: 750 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.820784:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513138000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993114752:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.820787:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f40d9167/0xd5401cbabc01fba3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x611:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06b4 expref: 750 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.820800:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0257e5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x614:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.820806:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f40d9167/0xd5401cbabc01fba3 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x611:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06b4 expref: 750 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.820821:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x611:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.820824:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x611:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.820829:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0257c9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x611:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.820830:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35911 00010000:00010000:2.0:1783520935.820842:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0257c9 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x611:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.820843:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351313b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+749:2388:x1870153002396160:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.820856:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.820865:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000000c7c4c37/0xd5401cbabc01fbb1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x613:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06d0 expref: 749 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.820876:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000000c7c4c37/0xd5401cbabc01fbb1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x613:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06d0 expref: 749 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.820878:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.820892:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0257c9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x611:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.820897:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000c7c4c37/0xd5401cbabc01fbb1 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x613:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06d0 expref: 749 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.820909:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513139500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+749:2389:x1870153002396032:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 159us (345us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.820912:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0257c9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x611:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.820920:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363fe56680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993113472:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.820921:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x613:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.820927:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35910 00010000:00010000:1.0:1783520935.820932:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x613:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.820936:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0257c9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x611:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.820938:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0257d7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x613:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.820961:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0257d7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x613:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.820975:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0257c9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x611:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.820987:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.820995:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943611038e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993115520:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.821008:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.821012:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.821018:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.821024:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0257c9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x611:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.821032:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.821036:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0257c9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x611:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.821038:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351313b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+747:2388:x1870153002396160:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 196us (412us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.821045:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.821056:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35911 00010000:00010000:2.0:1783520935.821058:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0257c9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x611:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.821074:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0257c9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x611:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.821078:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002396288 00000100:00100000:1.0:1783520935.821083:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002396288 00010000:00010000:2.0:1783520935.821085:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0257c9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x611:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.821091:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363fe56300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+253:3627:x1870152993113728:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3163us (3236us total) trans 4294975819 rc 0/0 00010000:00010000:1.0:1783520935.821103:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000df0b1cb8/0xd5401cbabc01fbb8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x614:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06de expref: 747 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.821103:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44317 00010000:00010000:2.0:1783520935.821104:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.821108:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.821112:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.821123:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993115520 00000100:00100000:0.0:1783520935.821128:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993115520 00000100:00100000:1.0:1783520935.821136:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35912 00000100:00100000:1.0:1783520935.821145:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5bb800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+747:2387:x1870153002396288:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.821155:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.821155:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44325 00010000:00010000:1.0:1783520935.821163:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000df0b1cb8/0xd5401cbabc01fbb8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x614:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06de expref: 747 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.821164:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94361103b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+253:3626:x1870152993115520:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.821180:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000df0b1cb8/0xd5401cbabc01fbb8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x614:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06de expref: 747 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.821180:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1560 count 1 00010000:00010000:0.0:1783520935.821192:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025801 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x618:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.821200:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000df0b1cb8/0xd5401cbabc01fbb8 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x614:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06de expref: 747 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.821205:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008fe56652/0xd5401cbabc01fbd4 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x618:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0716 expref: 747 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.821215:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943611038a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993115648:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.821221:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x614:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.821222:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x618:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.821231:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x614:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.821232:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x618:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.821244:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0257e5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x614:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.821245:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025801 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x618:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.821271:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0257e5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x614:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.821286:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008fe56652/0xd5401cbabc01fbd4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x618:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0716 expref: 746 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.821296:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.821304:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.821310:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008fe56652/0xd5401cbabc01fbd4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x618:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0716 expref: 746 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.821323:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530e680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993115776:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.821331:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5bb800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+746:2387:x1870153002396288:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 185us (348us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.821339:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35912 00000100:00100000:0.0:1783520935.821354:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470f00 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.821393:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0257e5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x614:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.821401:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0257d7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x613:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.821404:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363fe57100/2, rc 0, transno 4294975819 00010000:00010000:1.0:1783520935.821404:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0257e5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x614:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.821415:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.821418:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363fe57100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993113728:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.821423:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0257d7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x613:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.821432:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0257e5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x614:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.821438:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0257d7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x613:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.821445:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0257e5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x614:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.821454:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0257d7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x613:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.821470:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.821476:0:24276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002396416 00000100:00100000:0.0:1783520935.821481:0:24276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002396416 00000100:00100000:3.0:1783520935.821486:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1783520935.821508:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943505876300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993115904:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.821514:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.821521:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0257e5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x614:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00100000:0.0:1783520935.821521:0:24276:0:(ofd_dev.c:2709:ofd_rw_hpreq_check()) @@@ lustre-OST0001 ll_ost_io00_011: refresh rw locks for [0x2c0000401:0x8f7:0x0] (65536->262143) req@ffff94350c5b8000 x1870153002396416/t0(0) o4->f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff@192.168.204.33@tcp:301/0 lens 504/0 e 0 to 0 dl 1783520946 ref 1 fl New:/600/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00002000:00100000:0.0:1783520935.821540:0:24276:0:(ofd_dev.c:2565:ofd_prolong_extent_locks()) Prolong locks for req ffff94350c5b8000 with x1870153002396416 ext(65536->262143) 00010000:00010000:2.0:1783520935.821543:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9435122dd200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.821551:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0257e5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x614:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.821551:0:24276:0:(ldlm_extent.c:791:ldlm_lock_prolong_one()) ### refreshed to 100s. ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0250bb lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400000020 nid: 192.168.204.33@tcp remote: 0xc275876b859f34a4 expref: 746 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.821576:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+254:3626:x1870152993114496:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2285us (2600us total) trans 4294975820 rc 0/0 00010000:00010000:2.0:1783520935.821577:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0257e5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x614:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.821581:0:24276:0:(ldlm_lockd.c:607:ldlm_refresh_waiting_lock()) ### refreshed to 100s ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0250bb lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400000020 nid: 192.168.204.33@tcp remote: 0xc275876b859f34a4 expref: 746 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.821594:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44321 00002000:00010000:0.0:1783520935.821603:0:24276:0:(ofd_dev.c:2714:ofd_rw_hpreq_check()) lustre-OST0001: refreshed 1 locks timeout for req ffff94350c5b8000 00010000:00010000:2.0:1783520935.821611:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0257e5 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x614:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.821617:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993115648 00010000:00010000:2.0:1783520935.821621:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0257e5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x614:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.821623:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993115648 00000100:00100000:0.0:1783520935.821626:0:24276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 4848 00000100:00100000:3.0:1783520935.821631:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.821637:0:24276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5b8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_io00_011:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+746:2387:x1870153002396416:12345-192.168.204.33@tcp:4:dd.0 00010000:00010000:2.0:1783520935.821647:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.821651:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.821655:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.821661:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993115904 00000100:00100000:1.0:1783520935.821666:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993115904 00000004:00080000:3.0:1783520935.821687:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351530e300/2, rc 0, transno 4294975820 00000100:00100000:1.0:1783520935.821696:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44326 00000100:00100000:3.0:1783520935.821700:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993114496:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.821707:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505875f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+255:3627:x1870152993115648:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.821719:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b8380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993115264:192.168.204.33@tcp:104: 00002000:00080000:1.0:1783520935.821720:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1561 count 1 00010000:00010000:1.0:1783520935.821737:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025808 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x619:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.821740:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0257ec lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x615:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.821745:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1783520935.821758:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943505877100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993116032:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.821762:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000fd7648dd/0xd5401cbabc01fbdb lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x619:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0724 expref: 746 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.821771:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x619:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.821775:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x619:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.821776:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44327 00010000:00010000:1.0:1783520935.821783:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025808 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x619:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.821792:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505876680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+255:3627:x1870152993115904:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.821794:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00080000:2.0:1783520935.821812:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1562 count 1 00010000:00010000:1.0:1783520935.821817:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000fd7648dd/0xd5401cbabc01fbdb lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x619:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0724 expref: 746 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.821827:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02580f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x61a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.821831:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000fd7648dd/0xd5401cbabc01fbdb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x619:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0724 expref: 746 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.821837:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993116032 00000100:00100000:3.0:1783520935.821841:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993116032 00010000:00010000:2.0:1783520935.821856:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000595f0dad/0xd5401cbabc01fbe2 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x61a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0732 expref: 747 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.821857:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513138000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993116160:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.821865:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44328 00010000:00010000:2.0:1783520935.821874:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x61a:0x0].0x0 since=37s v=(0 0) 00000100:00100000:3.0:1783520935.821875:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e0f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+256:3626:x1870152993116032:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.821883:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x61a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.821886:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.821888:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02580f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x61a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520935.821890:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1563 count 1 00010000:00010000:3.0:1783520935.821903:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025816 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x61b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.821905:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57d80 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.821917:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000595f0dad/0xd5401cbabc01fbe2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x61a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0732 expref: 747 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.821918:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006407aff2/0xd5401cbabc01ff69 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x61b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0740 expref: 747 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.821935:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x61b:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.821940:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x61b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.821945:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000595f0dad/0xd5401cbabc01fbe2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x61a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0732 expref: 747 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.821946:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025816 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x61b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.821947:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000025a91e3c/0xd5401cbabc022ac0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x616:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31b0 expref: 747 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.821967:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514fdf480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+257:3625:x1870152993114112:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3533us (3638us total) trans 4294975821 rc 0/0 00000100:00100000:2.0:1783520935.821979:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b8380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993116288:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.821984:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44319 00010000:00010000:1.0:1783520935.821994:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000025a91e3c/0xd5401cbabc022ac0 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x616:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31b0 expref: 747 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.822007:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.822014:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0257d7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x613:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.822015:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513138380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993116416:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.822016:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676300 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.822021:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006407aff2/0xd5401cbabc01ff69 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x61b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0740 expref: 747 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.822039:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.822046:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006407aff2/0xd5401cbabc01ff69 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x61b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0740 expref: 747 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.822046:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0257d7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x613:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.822052:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6cc0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.822073:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0257d7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x613:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.822077:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514f99c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993116544:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.822095:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0257d7 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x613:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.822109:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0257d7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x613:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.822126:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.822128:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.822130:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.822356:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c553100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993115392:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.822445:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.822471:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002396672 00000100:00100000:1.0:1783520935.822477:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002396672 00000100:00100000:2.0:1783520935.822479:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.822512:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000999101c4/0xd5401cbabc01fbbf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x615:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06ec expref: 748 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.822515:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002396544 00000100:00100000:2.0:1783520935.822521:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002396544 00010000:00010000:2.0:1783520935.822543:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000048b513d5/0xd5401cbabc01fbcd lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x617:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0708 expref: 749 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.822547:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35913 00000100:00100000:1.0:1783520935.822560:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5bb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+749:2387:x1870153002396672:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.822570:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.822575:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000999101c4/0xd5401cbabc01fbbf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x615:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06ec expref: 749 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.822586:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000999101c4/0xd5401cbabc01fbbf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x615:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06ec expref: 749 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.822596:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35914 00010000:00010000:1.0:1783520935.822604:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000999101c4/0xd5401cbabc01fbbf lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x615:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f06ec expref: 749 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.822609:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5b8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+749:2387:x1870153002396544:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.822623:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x615:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.822624:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.822631:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x615:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.822633:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000048b513d5/0xd5401cbabc01fbcd lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x617:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0708 expref: 748 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.822636:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0257ec lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x615:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.822652:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0257ec lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x615:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.822653:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000048b513d5/0xd5401cbabc01fbcd lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x617:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0708 expref: 748 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.822679:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000048b513d5/0xd5401cbabc01fbcd lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x617:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0708 expref: 748 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.822682:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.822694:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x617:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.822698:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x617:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.822702:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0257fa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x617:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.822711:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0257fa lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x617:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.822718:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5bb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+747:2387:x1870153002396672:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 160us (272us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.822727:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35913 00010000:00010000:2.0:1783520935.822728:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.822753:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5b8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+746:2387:x1870153002396544:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 148us (277us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.822761:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35914 00010000:00010000:2.0:1783520935.822793:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0257ec lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x615:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.822803:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0257ec lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x615:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.822812:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0257ec lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x615:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.822820:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0257ec lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x615:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.822832:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.822844:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.822850:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0257ec lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x615:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.822850:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530e680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993115776:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.822864:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.822869:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0257ec lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x615:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.822874:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025801 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x618:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.822877:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0257ec lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x615:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.822904:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0257ec lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x615:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.822912:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+257:3627:x1870152993114880:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2754us (2895us total) trans 4294975822 rc 0/0 00010000:00010000:1.0:1783520935.822914:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0257ec lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x615:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.822927:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44322 00010000:00010000:1.0:1783520935.822946:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.822949:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.822951:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.822993:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.823016:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002396800 00000100:00100000:2.0:1783520935.823022:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002396800 00010000:00010000:2.0:1783520935.823038:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008fe56652/0xd5401cbabc01fbd4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x618:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0716 expref: 746 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.823061:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35915 00000100:00100000:2.0:1783520935.823071:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513139880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+746:2388:x1870153002396800:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.823079:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.823084:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008fe56652/0xd5401cbabc01fbd4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x618:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0716 expref: 746 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.823100:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008fe56652/0xd5401cbabc01fbd4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x618:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0716 expref: 746 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.823116:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008fe56652/0xd5401cbabc01fbd4 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x618:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0716 expref: 746 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.823125:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x618:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.823127:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x618:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.823130:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025801 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x618:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.823138:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025801 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x618:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.823161:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.823163:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513138000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993116160:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.823175:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025808 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x619:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.823182:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025801 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x618:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.823190:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513139880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+745:2388:x1870153002396800:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 120us (196us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.823199:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025801 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x618:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.823203:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35915 00010000:00010000:0.0:1783520935.823214:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025801 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x618:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.823220:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025801 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x618:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.823239:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.823306:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94361103b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+257:3626:x1870152993115520:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2145us (2266us total) trans 4294975823 rc 0/0 00000100:00100000:0.0:1783520935.823314:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44325 00010000:00010000:2.0:1783520935.823323:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.823329:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025801 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x618:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.823337:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.823341:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025801 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x618:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.823347:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943611038e00/2, rc 0, transno 4294975823 00010000:00010000:2.0:1783520935.823381:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025801 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x618:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.823393:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943611038e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993115520:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.823395:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025801 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x618:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.823406:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025801 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x618:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.823422:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.823425:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.823427:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.823440:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.823445:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993116672:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.823472:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.823540:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.823557:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993116672 00000100:00100000:1.0:1783520935.823561:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993116672 00000100:00100000:1.0:1783520935.823585:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44329 00000100:00100000:1.0:1783520935.823594:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+257:3626:x1870152993116672:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.823602:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00002000:00080000:1.0:1783520935.823612:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1564 count 1 00000100:00100000:2.0:1783520935.823622:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002396928 00010000:00010000:1.0:1783520935.823627:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc02581d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x61c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.823628:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002396928 00010000:00010000:1.0:1783520935.823642:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000071ea6dd/0xd5401cbabc01fbe9 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x61c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f074e expref: 745 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.823648:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000fd7648dd/0xd5401cbabc01fbdb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x619:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0724 expref: 745 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.823651:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x61c:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.823657:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x61c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.823662:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0257fa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x617:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.823664:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc02581d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x61c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.823675:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002397184 00000100:00100000:2.0:1783520935.823680:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002397184 00010000:00010000:2.0:1783520935.823696:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000025a91e3c/0xd5401cbabc022ac0 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x616:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31b0 expref: 746 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.823705:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000071ea6dd/0xd5401cbabc01fbe9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x61c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f074e expref: 746 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.823713:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002397056 00000100:00100000:2.0:1783520935.823717:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002397056 00010000:00010000:1.0:1783520935.823719:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000071ea6dd/0xd5401cbabc01fbe9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x61c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f074e expref: 746 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.823731:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000595f0dad/0xd5401cbabc01fbe2 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x61a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0732 expref: 747 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.823733:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513138000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993116800:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.823754:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35916 00000100:00100000:2.0:1783520935.823762:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638587450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+747:2387:x1870153002396928:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.823770:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.823774:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000fd7648dd/0xd5401cbabc01fbdb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x619:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0724 expref: 747 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.823782:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c369480 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.823786:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000fd7648dd/0xd5401cbabc01fbdb lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x619:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0724 expref: 747 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.823804:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35917 00010000:00010000:2.0:1783520935.823805:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000fd7648dd/0xd5401cbabc01fbdb lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x619:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0724 expref: 747 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.823817:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435079f1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+746:2387:x1870153002397184:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.823820:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x619:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.823824:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x619:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.823826:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.823829:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025808 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x619:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.823835:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000025a91e3c/0xd5401cbabc022ac0 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x616:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31b0 expref: 746 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.823842:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025808 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x619:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.823852:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000025a91e3c/0xd5401cbabc022ac0 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x616:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31b0 expref: 746 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.823863:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.823869:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x616:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.823875:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x616:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.823879:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc0257f3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x616:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.823886:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638587450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+746:2387:x1870153002396928:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 122us (445us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.823887:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0257fa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x617:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.823895:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc0257f3 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x616:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.823900:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0257fa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x617:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.823903:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35916 00010000:00010000:3.0:1783520935.823918:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0257fa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x617:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.823923:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.823926:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0257fa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x617:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.823939:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520935.823943:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35918 00000100:00100000:2.0:1783520935.823956:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517244e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+745:2388:x1870153002397056:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.823956:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435079f1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+745:2387:x1870153002397184:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 137us (411us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.823970:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.823971:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35917 00010000:00010000:2.0:1783520935.823979:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000595f0dad/0xd5401cbabc01fbe2 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x61a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0732 expref: 745 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.823990:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000595f0dad/0xd5401cbabc01fbe2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x61a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0732 expref: 744 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.823997:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025808 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x619:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.824003:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x61a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.824004:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025808 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x619:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.824012:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x61a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.824015:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025808 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x619:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.824016:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c550000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+258:3626:x1870152993115136:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3733us (3819us total) trans 4294975824 rc 0/0 00010000:00010000:2.0:1783520935.824018:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02580f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.824027:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025808 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x619:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.824033:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44324 00010000:00010000:2.0:1783520935.824035:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02580f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x61a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.824045:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000004:00080000:0.0:1783520935.824045:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c552d80/2, rc 0, transno 4294975824 00010000:00010000:2.0:1783520935.824049:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.824057:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c552d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993115136:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.824073:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517244e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+744:2388:x1870153002397056:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 122us (475us total) trans 0 rc 0/0 00000004:00080000:3.0:1783520935.824076:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351530c700/2, rc 0, transno 4294975822 00000100:00100000:3.0:1783520935.824082:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993114880:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.824082:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35918 00000100:00100000:3.0:1783520935.824095:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993116928:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.824103:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520935.824105:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.824108:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0257fa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x617:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.824110:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025808 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x619:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.824130:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9435122dc800 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:3.0:1783520935.824131:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.824140:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4600 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520935.824140:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505875f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+258:3627:x1870152993115648:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2432us (2835us total) trans 4294975825 rc 0/0 00010000:00010000:0.0:1783520935.824140:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025808 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x619:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.824147:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0257fa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x617:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.824157:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44326 00000004:00080000:3.0:1783520935.824158:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943611038a80/2, rc 0, transno 4294975825 00010000:00010000:0.0:1783520935.824167:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025808 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x619:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.824168:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0257fa lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x617:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.824171:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943611038a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993115648:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.824173:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993116928 00000100:00100000:1.0:1783520935.824179:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993116928 00000100:00100000:3.0:1783520935.824184:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1783520935.824194:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993117056:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.824196:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025808 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x619:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.824198:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0257fa lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x617:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.824208:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025808 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x619:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.824210:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44330 00000100:00100000:3.0:1783520935.824219:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.824221:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0257fa lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x617:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.824226:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514f9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+258:3627:x1870152993116928:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.824241:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00002000:00080000:1.0:1783520935.824245:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1565 count 1 00010000:00010000:0.0:1783520935.824246:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.824248:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.824251:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.824253:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.824256:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.824261:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025824 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x61d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.824266:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993117056 00000100:00100000:0.0:1783520935.824273:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993117056 00010000:00010000:1.0:1783520935.824277:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000328c58d9/0xd5401cbabc01fbf0 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x61d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f075c expref: 743 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.824284:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x61d:0x0].0x0 since=37s v=(0 0) 00000100:00100000:3.0:1783520935.824285:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cc7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993117184:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.824289:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000595f0dad/0xd5401cbabc01fbe2 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x61a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0732 expref: 743 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.824291:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x61d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.824301:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025824 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x61d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.824307:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44331 00000100:00100000:2.0:1783520935.824308:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b8380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993116288:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.824315:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.824318:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c553100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+259:3627:x1870152993117056:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.824326:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02580f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.824331:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1566 count 1 00010000:00010000:2.0:1783520935.824338:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02580f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.824342:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c5bbb80/2, rc 0, transno 4294975821 00010000:00010000:1.0:1783520935.824344:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000328c58d9/0xd5401cbabc01fbf0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x61d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f075c expref: 742 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.824344:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02582b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x61e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.824351:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993114112:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.824355:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02580f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.824394:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000328c58d9/0xd5401cbabc01fbf0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x61d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f075c expref: 742 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.824407:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351313a680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993117312:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.824432:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:0.0:1783520935.824440:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000232bfca6/0xd5401cbabc0229af lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x61e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f309f expref: 742 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.824443:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993117440:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.824452:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435131fce40 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.824454:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x61e:0x0].0x0 since=36s v=(0 0) 00010000:00010000:0.0:1783520935.824460:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x61e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.824465:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02582b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x61e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.824477:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.824504:0:12734:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000025a91e3c/0xd5401cbabc022ac0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x616:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31b0 expref: 742 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.824522:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993117184 00000100:00100000:1.0:1783520935.824522:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513138380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993116416:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.824528:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993117184 00010000:00010000:1.0:1783520935.824543:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc0257f3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x616:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.824554:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc0257f3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x616:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.824555:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993117440 00000100:00100000:3.0:1783520935.824560:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993117440 00010000:00010000:1.0:1783520935.824562:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc0257f3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x616:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.824574:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44332 00010000:00010000:1.0:1783520935.824581:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc0257f3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x616:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.824585:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02580f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.824591:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44333 00000100:00100000:0.0:1783520935.824591:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cc7ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+261:3625:x1870152993117184:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.824597:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc0257f3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x616:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.824603:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638995180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+261:3625:x1870152993117440:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.824607:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02580f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x61a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.824612:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1567 count 1 00002000:00080000:3.0:1783520935.824622:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1568 count 1 00010000:00010000:1.0:1783520935.824629:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:0.0:1783520935.824630:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025832 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x61f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.824634:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520935.824635:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025839 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x620:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.824642:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520935.824647:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.824652:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc0257f3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x616:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.824654:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02580f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x61a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.824659:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e855a946/0xd5401cbabc01fbfe lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x61f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0778 expref: 741 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.824660:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000000ac3930d/0xd5401cbabc01fc05 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x620:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0786 expref: 741 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.824677:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9435122dcc00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.824678:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.824684:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x620:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.824685:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc0257f3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x616:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.824686:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02580f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x61a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.824691:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x620:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.824701:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025839 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x620:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.824708:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x61f:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.824716:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x61f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.824717:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02580f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x61a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.824717:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc0257f3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x616:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.824722:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025832 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x61f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.824730:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000000ac3930d/0xd5401cbabc01fc05 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x620:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0786 expref: 741 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.824748:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02580f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x61a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.824752:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc0257f3 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x616:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.824754:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000000ac3930d/0xd5401cbabc01fc05 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x620:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0786 expref: 741 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.824756:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02580f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x61a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.824766:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e855a946/0xd5401cbabc01fbfe lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x61f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0778 expref: 741 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.824779:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc0257f3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x616:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.824785:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.824789:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.824793:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.824796:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e855a946/0xd5401cbabc01fbfe lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x61f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0778 expref: 741 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.824804:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943638997800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993117568:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.824815:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.824819:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.824820:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a4700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993117696:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.824823:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.824835:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967e40 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.824857:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470cc0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.824869:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435129679c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.824882:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000232bfca6/0xd5401cbabc0229af lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x61e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f309f expref: 741 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.824883:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505876680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+263:3627:x1870152993115904:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3096us (3343us total) trans 4294975826 rc 0/0 00000004:00080000:3.0:1783520935.824895:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943505876300/2, rc 0, transno 4294975826 00000100:00100000:2.0:1783520935.824899:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44327 00010000:00010000:0.0:1783520935.824899:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000232bfca6/0xd5401cbabc0229af lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x61e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f309f expref: 741 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.824904:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943505876300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993115904:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.824917:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9ce00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993117824:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.824920:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+262:3627:x1870152993115008:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4653us (4822us total) trans 4294975827 rc 0/0 00000004:00080000:3.0:1783520935.824924:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351530fb80/2, rc 0, transno 4294975827 00000100:00100000:1.0:1783520935.824931:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44323 00000100:00100000:3.0:1783520935.824932:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993115008:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.824936:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470f00 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.824986:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943638996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993117952:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.825008:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.825024:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1783520935.825029:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943638996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993118080:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.825038:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993117952 00000100:00100000:2.0:1783520935.825042:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993117952 00000100:00100000:3.0:1783520935.825048:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.825063:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993118080 00000100:00100000:2.0:1783520935.825066:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993118080 00000100:00100000:1.0:1783520935.825082:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44334 00000100:00100000:2.0:1783520935.825089:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44335 00000100:00100000:1.0:1783520935.825094:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638995500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+263:3627:x1870152993117952:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.825102:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350d165880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+263:3627:x1870152993118080:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.825109:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1569 count 1 00002000:00080000:2.0:1783520935.825117:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1570 count 1 00010000:00010000:1.0:1783520935.825123:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc025840 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x621:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.825132:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025847 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x622:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.825138:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006ba56c76/0xd5401cbabc01fc0c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x621:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0794 expref: 741 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.825150:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x621:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.825151:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d7ab05d1/0xd5401cbabc01fc13 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x622:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f07a2 expref: 741 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.825156:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x621:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.825165:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc025840 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x621:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.825166:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x622:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.825174:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x622:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.825183:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025847 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x622:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.825201:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006ba56c76/0xd5401cbabc01fc0c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x621:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0794 expref: 741 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.825211:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d7ab05d1/0xd5401cbabc01fc13 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x622:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07a2 expref: 741 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.825226:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006ba56c76/0xd5401cbabc01fc0c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x621:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0794 expref: 741 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.825242:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d7ab05d1/0xd5401cbabc01fc13 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x622:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07a2 expref: 741 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.825248:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513138380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993118208:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.825263:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b8380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993118336:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.825290:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6cc0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.825294:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676300 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.825598:0:24276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5b8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_io00_011:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+742:2387:x1870153002396416:12345-192.168.204.33@tcp:4:dd.0 Request processed in 3962us (4187us total) trans 4294975828 rc 0/0 00000100:00100000:0.0:1783520935.825610:0:24276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 4848 00002000:00100000:0.0:1783520935.825617:0:24276:0:(ofd_dev.c:2709:ofd_rw_hpreq_check()) @@@ lustre-OST0001 ll_ost_io00_011: refresh rw locks for [0x2c0000401:0x8f7:0x0] (65536->262143) req@ffff94350c5b8000 x1870153002396416/t4294975828(0) o4->f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff@192.168.204.33@tcp:301/0 lens 504/448 e 0 to 0 dl 1783520946 ref 1 fl Complete:H/600/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00002000:00100000:0.0:1783520935.825627:0:24276:0:(ofd_dev.c:2565:ofd_prolong_extent_locks()) Prolong locks for req ffff94350c5b8000 with x1870153002396416 ext(65536->262143) 00010000:00010000:0.0:1783520935.825635:0:24276:0:(ldlm_extent.c:791:ldlm_lock_prolong_one()) ### refreshed to 100s. ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0250bb lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400000020 nid: 192.168.204.33@tcp remote: 0xc275876b859f34a4 expref: 742 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.825647:0:24276:0:(ldlm_lockd.c:607:ldlm_refresh_waiting_lock()) ### refreshed to 100s ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0250bb lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400000020 nid: 192.168.204.33@tcp remote: 0xc275876b859f34a4 expref: 742 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00010000:0.0:1783520935.825655:0:24276:0:(ofd_dev.c:2714:ofd_rw_hpreq_check()) lustre-OST0001: refreshed 1 locks timeout for req ffff94350c5b8000 00000100:00100000:1.0:1783520935.826235:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513138000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993116800:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.826251:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc02581d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x61c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.826260:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943638997800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993117568:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.826279:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025839 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x620:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.826279:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351313a680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993117312:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.826293:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025824 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x61d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.826326:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a4700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993117696:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.826344:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025832 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x61f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.826483:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514f99c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993116544:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.826513:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025816 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x61b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.826535:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9ce00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993117824:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.826554:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02582b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x61e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.826715:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513138380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993118208:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.826730:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc025840 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x621:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.826910:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b8380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993118336:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.826922:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025847 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x622:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.827135:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.827170:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.827188:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002397440 00000100:00100000:2.0:1783520935.827195:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002397440 00000100:00100000:1.0:1783520935.827212:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002397696 00000100:00100000:0.0:1783520935.827218:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.827219:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000328c58d9/0xd5401cbabc01fbf0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x61d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f075c expref: 742 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.827219:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002397696 00010000:00010000:1.0:1783520935.827243:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e855a946/0xd5401cbabc01fbfe lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x61f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0778 expref: 743 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.827255:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.827256:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002397952 00000100:00100000:2.0:1783520935.827261:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002397952 00000100:00100000:1.0:1783520935.827272:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002398208 00010000:00010000:2.0:1783520935.827278:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000232bfca6/0xd5401cbabc0229af lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x61e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f309f expref: 744 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.827278:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002398208 00000100:00100000:0.0:1783520935.827292:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35919 00010000:00010000:1.0:1783520935.827297:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d7ab05d1/0xd5401cbabc01fc13 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x622:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07a2 expref: 745 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.827306:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9ce00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+745:2387:x1870153002397440:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.827317:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35920 00010000:00010000:0.0:1783520935.827322:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.827327:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35921 00010000:00010000:0.0:1783520935.827331:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000328c58d9/0xd5401cbabc01fbf0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x61d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f075c expref: 745 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.827333:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9ea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+745:2387:x1870153002397696:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.827345:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.827346:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9d500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+745:2387:x1870153002397952:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.827355:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e855a946/0xd5401cbabc01fbfe lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x61f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0778 expref: 745 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.827390:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.827396:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e855a946/0xd5401cbabc01fbfe lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x61f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0778 expref: 745 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.827399:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000328c58d9/0xd5401cbabc01fbf0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x61d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f075c expref: 745 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.827400:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000232bfca6/0xd5401cbabc0229af lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x61e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f309f expref: 745 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.827429:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000232bfca6/0xd5401cbabc0229af lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x61e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f309f expref: 745 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.827435:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e855a946/0xd5401cbabc01fbfe lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x61f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0778 expref: 745 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.827439:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000328c58d9/0xd5401cbabc01fbf0 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x61d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f075c expref: 745 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.827454:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000232bfca6/0xd5401cbabc0229af lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x61e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f309f expref: 745 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.827465:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x61f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.827468:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x61d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.827472:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x61f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.827472:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x61e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.827473:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x61d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.827482:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x61e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.827486:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025832 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.827486:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025824 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.827489:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02582b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.827524:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025832 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x61f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.827526:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025824 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x61d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.827527:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02582b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x61e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.827566:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.827571:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.827576:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.827604:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9ea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+742:2387:x1870153002397696:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 275us (431us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.827604:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9ce00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+742:2387:x1870153002397440:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 301us (467us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.827611:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9d500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+742:2387:x1870153002397952:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 268us (391us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.827621:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35919 00000100:00100000:2.0:1783520935.827629:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35920 00000100:00100000:1.0:1783520935.827638:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35921 00000100:00100000:0.0:1783520935.827646:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35922 00000100:00100000:0.0:1783520935.827657:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9df80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+739:2387:x1870153002398208:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.827668:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.827673:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d7ab05d1/0xd5401cbabc01fc13 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x622:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07a2 expref: 739 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.827684:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d7ab05d1/0xd5401cbabc01fc13 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x622:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07a2 expref: 739 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.827696:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025824 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.827701:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d7ab05d1/0xd5401cbabc01fc13 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x622:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07a2 expref: 739 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.827717:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025824 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.827721:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x622:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.827729:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x622:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.827734:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025824 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.827736:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025847 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x622:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.827748:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.827758:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025824 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x61d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.827760:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025847 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x622:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.827770:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002397312 00000100:00100000:2.0:1783520935.827776:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002397312 00010000:00010000:1.0:1783520935.827789:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.827797:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.827798:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000071ea6dd/0xd5401cbabc01fbe9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x61c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f074e expref: 739 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.827829:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35923 00000100:00100000:0.0:1783520935.827829:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9df80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+739:2387:x1870153002398208:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 171us (573us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.827842:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35922 00000100:00100000:2.0:1783520935.827846:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638586c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+739:2387:x1870153002397312:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.827856:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.827861:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000071ea6dd/0xd5401cbabc01fbe9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x61c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f074e expref: 738 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.827865:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.827873:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025824 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x61d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.827879:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000071ea6dd/0xd5401cbabc01fbe9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x61c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f074e expref: 738 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.827904:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9435122dc600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.827909:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025824 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x61d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.827912:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000071ea6dd/0xd5401cbabc01fbe9 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x61c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f074e expref: 738 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.827922:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514f9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+264:3627:x1870152993116928:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3700us (3790us total) trans 4294975829 rc 0/0 00010000:00010000:0.0:1783520935.827932:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025824 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x61d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.827939:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44330 00010000:00010000:2.0:1783520935.827940:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x61c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.827948:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x61c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.827952:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc02581d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.827957:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025824 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x61d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.827958:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc02581d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x61c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.827980:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351530ca80/2, rc 0, transno 4294975829 00010000:00010000:2.0:1783520935.827984:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.827986:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025824 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x61d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.827993:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.827996:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993116928:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.828018:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638586c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+737:2387:x1870153002397312:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 176us (273us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.828019:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.828023:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.828027:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.828035:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35923 00010000:00010000:0.0:1783520935.828043:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025832 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.828047:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.828057:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025832 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.828066:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025832 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.828068:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1783520935.828072:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002397568 00000100:00100000:2.0:1783520935.828077:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002397824 00000100:00100000:1.0:1783520935.828077:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002397568 00000100:00100000:3.0:1783520935.828084:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350d166680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993118464:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.828085:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002397824 00010000:00010000:0.0:1783520935.828091:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025832 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x61f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.828096:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000000ac3930d/0xd5401cbabc01fc05 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x620:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0786 expref: 738 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.828108:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006407aff2/0xd5401cbabc01ff69 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x61b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0740 expref: 738 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.828121:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520935.828123:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.828131:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35924 00000100:00100000:1.0:1783520935.828139:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513138380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+738:2387:x1870153002397568:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.828147:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520935.828148:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35925 00010000:00010000:1.0:1783520935.828153:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000000ac3930d/0xd5401cbabc01fc05 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x620:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0786 expref: 738 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.828163:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351313a680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+738:2387:x1870153002397824:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.828164:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000000ac3930d/0xd5401cbabc01fc05 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x620:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0786 expref: 738 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.828174:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.828180:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006407aff2/0xd5401cbabc01ff69 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x61b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0740 expref: 738 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.828181:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000ac3930d/0xd5401cbabc01fc05 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x620:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0786 expref: 738 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.828203:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006407aff2/0xd5401cbabc01ff69 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x61b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0740 expref: 737 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.828206:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x620:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.828215:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x620:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.828221:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025839 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x620:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.828222:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cc7ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+264:3625:x1870152993117184:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3635us (3906us total) trans 4294975830 rc 0/0 00010000:00010000:2.0:1783520935.828223:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006407aff2/0xd5401cbabc01ff69 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x61b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0740 expref: 737 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.828224:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cc7c700/2, rc 0, transno 4294975830 00000100:00100000:3.0:1783520935.828235:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cc7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993117184:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.828242:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44332 00010000:00010000:2.0:1783520935.828252:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x61b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.828256:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x61b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.828259:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025816 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.828264:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025816 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x61b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.828265:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025839 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x620:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.828268:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993118464 00000100:00100000:0.0:1783520935.828275:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993118464 00000100:00100000:3.0:1783520935.828287:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350d167100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993118592:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.828302:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.828307:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44336 00000100:00100000:0.0:1783520935.828316:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350d166a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+264:3626:x1870152993118464:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.828321:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.828326:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351313a680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+736:2387:x1870153002397824:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 165us (282us total) trans 0 rc 0/0 00002000:00080000:0.0:1783520935.828340:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1571 count 1 00010000:00010000:0.0:1783520935.828400:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02584e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x623:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.828404:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993118592 00000100:00100000:3.0:1783520935.828409:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993118592 00010000:00010000:0.0:1783520935.828417:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b00bb13f/0xd5401cbabc02009d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x623:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f07b0 expref: 736 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.828418:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.828424:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025839 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x620:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.828428:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x623:0x0].0x0 since=37s v=(0 0) 00000100:00100000:2.0:1783520935.828430:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35925 00010000:00010000:0.0:1783520935.828437:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x623:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.828440:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025839 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x620:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.828443:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02584e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x623:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.828449:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025839 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x620:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.828452:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513138380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+735:2387:x1870153002397568:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 310us (458us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.828457:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.828466:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025832 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x61f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.828468:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025839 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x620:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.828470:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35924 00010000:00010000:0.0:1783520935.828482:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b00bb13f/0xd5401cbabc02009d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x623:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07b0 expref: 735 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.828491:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.828508:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.828509:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025832 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x61f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.828523:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.828531:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b00bb13f/0xd5401cbabc02009d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x623:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07b0 expref: 734 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.828532:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025832 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x61f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.828555:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9f800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993118720:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.828565:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025832 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x61f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.828571:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.828580:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025832 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x61f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.828601:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002398080 00010000:00010000:2.0:1783520935.828605:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520935.828606:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002398080 00010000:00010000:2.0:1783520935.828610:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025839 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x620:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.828611:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470cc0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.828613:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638995180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+266:3625:x1870152993117440:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4009us (4133us total) trans 4294975831 rc 0/0 00010000:00010000:1.0:1783520935.828628:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006ba56c76/0xd5401cbabc01fc0c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x621:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0794 expref: 735 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.828631:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002398336 00000100:00100000:3.0:1783520935.828632:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44333 00000100:00100000:0.0:1783520935.828638:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002398336 00010000:00010000:2.0:1783520935.828646:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.828651:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025839 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x620:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.828660:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025839 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x620:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.828661:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0250bb lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400000020 nid: 192.168.204.33@tcp remote: 0xc275876b859f34a4 expref: 736 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.828668:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35926 00000100:00100000:1.0:1783520935.828682:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513138000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+736:2387:x1870153002398080:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.828685:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025839 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x620:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.828693:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.828695:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025839 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x620:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.828701:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006ba56c76/0xd5401cbabc01fc0c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x621:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0794 expref: 736 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.828702:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35927 00000004:00080000:3.0:1783520935.828710:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cba1f80/2, rc 0, transno 4294975831 00000100:00100000:0.0:1783520935.828714:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513139500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+736:2388:x1870153002398336:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.828725:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.828726:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993117440:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.828726:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.828730:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.828730:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006ba56c76/0xd5401cbabc01fc0c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x621:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0794 expref: 736 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.828736:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00002000:00010000:0.0:1783520935.828738:0:17759:0:(ofd_lvb.c:331:ofd_lvbo_update()) res: [0x2c0000401:0x8f7:0x0] updating lvb blocks from disk: 1 -> 256 00010000:00010000:0.0:1783520935.828747:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0250bb lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400000020 nid: 192.168.204.33@tcp remote: 0xc275876b859f34a4 expref: 736 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.828755:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006ba56c76/0xd5401cbabc01fc0c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x621:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0794 expref: 736 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.828757:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.828757:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0250bb lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400000020 nid: 192.168.204.33@tcp remote: 0xc275876b859f34a4 expref: 736 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.828780:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350d166d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993118848:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.828785:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025816 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.828785:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x621:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.828787:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0250bb lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400000020 nid: 192.168.204.33@tcp remote: 0xc275876b859f34a4 expref: 735 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.828796:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x621:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.828808:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc025840 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x621:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.828813:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.828814:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025816 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.828832:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f7:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.828836:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc025840 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x621:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.828839:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f7:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.828842:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025816 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.828848:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0257de lrc: 3/1,0 mode: PR/PR res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 3 type: EXT [0->266239] (req 0->266239) gid 0 flags: 0x40090000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.828865:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993118848 00010000:00010000:0.0:1783520935.828868:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0257de lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 3 type: EXT [0->266239] (req 0->266239) gid 0 flags: 0x40090000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.828872:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993118848 00010000:00010000:1.0:1783520935.828872:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.828887:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025816 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x61b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.828896:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.828897:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44337 00000100:00100000:1.0:1783520935.828897:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513138000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+734:2387:x1870153002398080:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 216us (374us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.828911:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350d165f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+266:3626:x1870152993118592:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.828915:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513139500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+734:2388:x1870153002398336:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 202us (344us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.828917:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35926 00010000:00010000:2.0:1783520935.828918:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.828927:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35927 00002000:00080000:3.0:1783520935.828929:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1572 count 1 00010000:00010000:1.0:1783520935.828943:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.828944:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025855 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x624:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.828950:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025816 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x61b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.828951:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44338 00000100:00100000:0.0:1783520935.828962:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638997800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+267:3627:x1870152993118848:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.828970:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009062b761/0xd5401cbabc0200a4 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x624:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f07be expref: 732 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.828973:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.828980:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025816 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x61b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.828982:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1573 count 1 00010000:00010000:3.0:1783520935.828983:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x624:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.828992:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x624:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.828998:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025816 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x61b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.828999:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025855 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x624:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.829000:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02585c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x625:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.829013:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e0f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+267:3626:x1870152993116032:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7137us (7221us total) trans 4294975832 rc 0/0 00010000:00010000:1.0:1783520935.829030:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025816 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x61b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.829031:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44328 00010000:00010000:0.0:1783520935.829035:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000fd9544ad/0xd5401cbabc0200ab lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x625:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f07cc expref: 732 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.829044:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009062b761/0xd5401cbabc0200a4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x624:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07be expref: 732 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.829054:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025816 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x61b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.829058:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc02581d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.829059:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x625:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.829070:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x625:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.829074:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009062b761/0xd5401cbabc0200a4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x624:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07be expref: 732 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.829079:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02585c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x625:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.829086:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc02581d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.829088:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.829092:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.829096:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.829108:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350544f100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993118976:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.829116:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc02581d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.829117:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc025840 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x621:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.829130:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000fd9544ad/0xd5401cbabc0200ab lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x625:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07cc expref: 732 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.829132:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc02581d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x61c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.829134:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967e40 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.829140:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc025840 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x621:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.829161:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943505877100/2, rc 0, transno 4294975832 00010000:00010000:0.0:1783520935.829162:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000fd9544ad/0xd5401cbabc0200ab lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x625:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07cc expref: 732 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.829166:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.829171:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc025840 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x621:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.829175:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943505877100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993116032:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.829190:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993119104:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.829191:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc025840 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x621:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.829208:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1783520935.829218:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520935.829223:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:0.0:1783520935.829224:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc02581d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x61c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.829234:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350544d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993119232:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.829234:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+267:3626:x1870152993116672:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5639us (5760us total) trans 4294975833 rc 0/0 00010000:00010000:0.0:1783520935.829238:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9435122dde00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.829246:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc02581d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x61c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.829249:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44329 00010000:00010000:0.0:1783520935.829260:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc02581d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x61c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.829270:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.829272:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.829275:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc02581d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x61c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.829278:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc025840 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x621:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.829290:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc02581d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x61c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.829292:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9435122dd600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.829297:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc025840 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x621:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.829309:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638995500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+267:3627:x1870152993117952:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4216us (4296us total) trans 4294975834 rc 0/0 00000004:00080000:3.0:1783520935.829312:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351530e680/2, rc 0, transno 4294975833 00010000:00010000:2.0:1783520935.829320:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc025840 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x621:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.829325:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993116672:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.829327:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44334 00010000:00010000:0.0:1783520935.829339:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.829343:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.829346:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc025840 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x621:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.829347:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.829351:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993119232 00000100:00100000:1.0:1783520935.829381:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993119232 00010000:00010000:0.0:1783520935.829386:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02582b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.829390:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc025840 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x621:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.829397:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:0.0:1783520935.829403:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02582b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.829411:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350544ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993119360:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.829412:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44339 00010000:00010000:2.0:1783520935.829416:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.829420:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.829425:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.829425:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02582b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x61e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.829426:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350544d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+267:3627:x1870152993119232:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.829441:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02582b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x61e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.829446:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025847 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x622:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.829446:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1574 count 1 00000100:00100000:3.0:1783520935.829447:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.829462:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025863 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x626:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.829465:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025847 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x622:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.829474:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.829478:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000202a1452/0xd5401cbabc02289e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x626:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f8e expref: 732 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.829487:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025847 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x622:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.829488:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943638996680/2, rc 0, transno 4294975834 00010000:00010000:1.0:1783520935.829489:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x626:0x0].0x0 since=36s v=(0 0) 00010000:00010000:1.0:1783520935.829510:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x626:0x0].0x0 since=0s v=(1 0) 00000100:00100000:3.0:1783520935.829515:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943638996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993117952:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.829519:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025863 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x626:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.829533:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025847 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x622:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.829557:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.829562:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02582b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x61e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.829563:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350544f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993119488:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.829566:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000202a1452/0xd5401cbabc02289e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x626:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f8e expref: 732 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.829578:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c553100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+268:3627:x1870152993117056:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5260us (5358us total) trans 4294975835 rc 0/0 00010000:00010000:2.0:1783520935.829587:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.829594:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000202a1452/0xd5401cbabc02289e lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x626:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f8e expref: 732 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.829595:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02582b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x61e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.829596:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44331 00000100:00100000:3.0:1783520935.829600:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.829619:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02582b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x61e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.829625:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351313b100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993119616:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.829634:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02582b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x61e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.829639:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435121a6d80/2, rc 0, transno 4294975835 00010000:00010000:2.0:1783520935.829641:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02582b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x61e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.829646:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993119360 00000100:00100000:3.0:1783520935.829653:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993119360 00000100:00100000:0.0:1783520935.829655:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993117056:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.829657:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.829662:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.829665:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.829673:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6cc0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.829678:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993119488 00010000:00010000:0.0:1783520935.829682:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0257de lrc: 3/1,0 mode: PR/PR res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 2 type: EXT [0->266239] (req 0->266239) gid 0 flags: 0x40010000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.829683:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993119488 00010000:00010000:0.0:1783520935.829695:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0257de lrc: 3/1,0 mode: PR/PR res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 2 type: EXT [0->266239] (req 0->266239) gid 0 flags: 0x40010000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.829700:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:0.0:1783520935.829707:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0257de lrc: 3/1,0 mode: PR/PR res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 2 type: EXT [0->266239] (req 0->266239) gid 0 flags: 0x50010000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.829713:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350544fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993119744:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.829717:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44340 00010000:00010000:0.0:1783520935.829722:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0257de lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 2 type: EXT [0->266239] (req 0->266239) gid 0 flags: 0x50010400000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.829731:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350544d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+269:3626:x1870152993119488:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.829732:0:9151:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0257de lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 2 type: EXT [0->266239] (req 0->266239) gid 0 flags: 0x40010400000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.829744:0:9151:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeba00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:3.0:1783520935.829748:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00080000:2.0:1783520935.829749:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1576 count 1 00010000:00010000:0.0:1783520935.829749:0:9151:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0257de lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 2 type: EXT [0->266239] (req 0->266239) gid 0 flags: 0x40010400000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.829762:0:9151:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0257de lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 2 type: EXT [0->266239] (req 0->266239) gid 0 flags: 0x40018400000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.829765:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02586a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x628:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.829772:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993119744 00010000:00010000:0.0:1783520935.829776:0:9151:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0257de lrc: 2/0,0 mode: --/PR res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 2 type: EXT [0->266239] (req 0->266239) gid 0 flags: 0x44819400000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.829778:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993119744 00010000:00010000:2.0:1783520935.829784:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000bb7c00da/0xd5401cbabc0200c0 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x628:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f07f6 expref: 732 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.829789:0:9151:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0257de lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 2 type: EXT [0->266239] (req 0->266239) gid 0 flags: 0x44819400000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.829798:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44341 00010000:00010000:2.0:1783520935.829802:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x628:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.829811:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x628:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.829812:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350544e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+270:3626:x1870152993119360:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.829821:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02586a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x628:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.829822:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44342 00002000:00080000:3.0:1783520935.829829:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1575 count 1 00000100:00100000:1.0:1783520935.829836:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350544df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+270:3628:x1870152993119744:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.829845:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025871 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x627:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.829853:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1577 count 1 00010000:00010000:2.0:1783520935.829854:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000bb7c00da/0xd5401cbabc0200c0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x628:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07f6 expref: 732 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.829863:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000abd9d0c6/0xd5401cbabc02013e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x627:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f07e8 expref: 732 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.829872:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc025878 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x629:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.829873:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000bb7c00da/0xd5401cbabc0200c0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x628:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07f6 expref: 732 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.829875:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x627:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.829885:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x627:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.829892:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635601c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:25342ed3-dd17-4439-8398-d8002e9f9f99+31:251800:x1870153002395648:12345-192.168.204.33@tcp:7:checkfiemap.0 Request processed in 10698us (11119us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.829896:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025871 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x627:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.829904:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000441e0c6f/0xd5401cbabc0200b9 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x629:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0804 expref: 732 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.829906:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b8380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993119872:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.829910:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 44320 00010000:00010000:1.0:1783520935.829923:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x629:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.829927:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x629:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.829931:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc025878 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x629:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.829938:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000abd9d0c6/0xd5401cbabc02013e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x627:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07e8 expref: 732 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.829944:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676a80 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.829953:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000abd9d0c6/0xd5401cbabc02013e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x627:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07e8 expref: 732 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.829961:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000441e0c6f/0xd5401cbabc0200b9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x629:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0804 expref: 732 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.829969:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350544ce00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993120000:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.829976:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.829982:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000441e0c6f/0xd5401cbabc0200b9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x629:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0804 expref: 732 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.829994:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435129679c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.829997:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513139880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993120128:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.830013:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.830019:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025847 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x622:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.830020:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470780 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.830030:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.830035:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025847 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x622:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.830044:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025847 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x622:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.830057:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025847 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x622:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.830064:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025847 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x622:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.830074:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350d165880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+271:3627:x1870152993118080:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4975us (5023us total) trans 4294975836 rc 0/0 00000100:00100000:2.0:1783520935.830084:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44335 00010000:00010000:1.0:1783520935.830087:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520935.830088:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943638996a00/2, rc 0, transno 4294975836 00010000:00010000:1.0:1783520935.830092:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.830096:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.830102:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943638996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993118080:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.830125:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435131fce40 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.830150:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350544dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993120256:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.830182:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.830311:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993120256 00000100:00100000:2.0:1783520935.830317:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993120256 00000100:00100000:2.0:1783520935.830349:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44343 00000100:00100000:2.0:1783520935.830383:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350544ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+271:3628:x1870152993120256:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.830396:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1578 count 1 00010000:00010000:2.0:1783520935.830409:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02587f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x62a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.830423:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0200ea lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x62a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0812 expref: 732 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.830430:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x62a:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.830434:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x62a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.830437:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02587f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x62a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.830460:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0200ea lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x62a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0812 expref: 732 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.830470:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0200ea lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x62a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0812 expref: 732 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.830480:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5bb800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993120384:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.830535:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676780 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.831628:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.831669:0:6497:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002398592 00000100:00100000:1.0:1783520935.831675:0:6497:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002398592 00000100:00100000:1.0:1783520935.831720:0:6497:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 9626 00000100:00100000:1.0:1783520935.831732:0:6497:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5bb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_rdpg00_001:25342ed3-dd17-4439-8398-d8002e9f9f99+13:251800:x1870153002398592:12345-192.168.204.33@tcp:35:checkfiemap.0 00000100:00100000:1.0:1783520935.831913:0:6497:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5bb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt_rdpg00_001:25342ed3-dd17-4439-8398-d8002e9f9f99+14:251800:x1870153002398592:12345-192.168.204.33@tcp:35:checkfiemap.0 Request processed in 184us (287us total) trans 8589957897 rc 0/0 00000100:00100000:1.0:1783520935.831919:0:6497:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 9626 00000100:00100000:2.0:1783520935.833158:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b8380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993119872:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.833161:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350544ce00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993120000:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.833187:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02586a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x628:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.833192:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025871 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x627:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.833226:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9f800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993118720:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.833241:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02584e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x623:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.833261:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5bb800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993120384:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.833273:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993119104:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.833277:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02587f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x62a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.833289:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02585c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x625:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.833296:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350544f100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993118976:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.833302:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.833319:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025855 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x624:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.833322:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.833340:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.833426:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.833429:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002398976 00000100:00100000:2.0:1783520935.833435:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002398976 00000100:00100000:1.0:1783520935.833455:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.833460:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000abd9d0c6/0xd5401cbabc02013e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x627:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07e8 expref: 733 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.833482:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.833491:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002399360 00000100:00100000:0.0:1783520935.833492:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002398720 00000100:00100000:2.0:1783520935.833506:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002399360 00000100:00100000:0.0:1783520935.833509:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002398720 00000100:00100000:1.0:1783520935.833514:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002399104 00000100:00100000:1.0:1783520935.833520:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002399104 00010000:00010000:2.0:1783520935.833524:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000441e0c6f/0xd5401cbabc0200b9 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x629:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0804 expref: 735 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.833538:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002399488 00010000:00010000:0.0:1783520935.833539:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000202a1452/0xd5401cbabc02289e lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x626:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f8e expref: 736 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.833543:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002399488 00010000:00010000:1.0:1783520935.833545:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b00bb13f/0xd5401cbabc02009d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x623:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07b0 expref: 736 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.833565:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009062b761/0xd5401cbabc0200a4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x624:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07be expref: 737 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.833585:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35928 00000100:00100000:1.0:1783520935.833587:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002398848 00000100:00100000:2.0:1783520935.833592:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35929 00000100:00100000:1.0:1783520935.833593:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002398848 00000100:00100000:0.0:1783520935.833597:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363fe55c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+737:2389:x1870153002398976:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.833608:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363fe56d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+738:2390:x1870153002399360:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.833609:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.833620:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000bb7c00da/0xd5401cbabc0200c0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x628:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07f6 expref: 738 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.833620:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000abd9d0c6/0xd5401cbabc02013e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x627:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07e8 expref: 738 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.833621:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.833631:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000441e0c6f/0xd5401cbabc0200b9 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x629:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0804 expref: 738 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.833663:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35930 00010000:00010000:0.0:1783520935.833668:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000abd9d0c6/0xd5401cbabc02013e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x627:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07e8 expref: 738 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.833673:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000441e0c6f/0xd5401cbabc0200b9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x629:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0804 expref: 738 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.833676:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638585850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+738:2387:x1870153002398720:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.833692:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.833698:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000202a1452/0xd5401cbabc02289e lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x626:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f8e expref: 738 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.833703:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x629:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.833704:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000abd9d0c6/0xd5401cbabc02013e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x627:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07e8 expref: 738 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.833712:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x629:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.833719:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc025878 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x629:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.833720:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x627:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.833723:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000202a1452/0xd5401cbabc02289e lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x626:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f8e expref: 737 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.833728:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x627:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.833739:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025871 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x627:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.833741:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc025878 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x629:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.833748:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x626:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.833756:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x626:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.833764:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025871 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x627:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.833765:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025863 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x626:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.833776:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.833787:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025863 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x626:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.833808:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.833813:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.833814:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363fe56d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+737:2390:x1870153002399360:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 209us (489us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.833823:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35929 00000100:00100000:1.0:1783520935.833839:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638585850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+736:2387:x1870153002398720:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 164us (494us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.833847:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35931 00000100:00100000:0.0:1783520935.833848:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363fe55c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+736:2389:x1870153002398976:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 251us (547us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.833859:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35930 00000100:00100000:2.0:1783520935.833861:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363fe56300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+736:2387:x1870153002399104:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.833868:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35928 00010000:00010000:2.0:1783520935.833875:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.833883:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b00bb13f/0xd5401cbabc02009d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x623:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07b0 expref: 734 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.833883:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35932 00000100:00100000:1.0:1783520935.833895:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363fe55180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+734:2387:x1870153002399488:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.833903:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.833904:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b00bb13f/0xd5401cbabc02009d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x623:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07b0 expref: 734 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.833910:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009062b761/0xd5401cbabc0200a4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x624:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07be expref: 734 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.833921:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35933 00000100:00100000:0.0:1783520935.833934:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e004e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+734:2387:x1870153002398848:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.833936:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009062b761/0xd5401cbabc0200a4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x624:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07be expref: 734 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.833943:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.833944:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b00bb13f/0xd5401cbabc02009d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x623:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07b0 expref: 734 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.833952:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000bb7c00da/0xd5401cbabc0200c0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x628:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07f6 expref: 734 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.833969:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x623:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.833972:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009062b761/0xd5401cbabc0200a4 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x624:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07be expref: 733 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.833973:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x623:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.833982:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02584e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x623:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.833988:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000bb7c00da/0xd5401cbabc0200c0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x628:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07f6 expref: 733 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.833996:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x624:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.834002:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02584e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x623:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.834006:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x624:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.834017:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025855 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x624:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.834022:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000bb7c00da/0xd5401cbabc0200c0 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x628:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07f6 expref: 732 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.834036:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025855 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x624:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.834038:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.834044:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x628:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.834048:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x628:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.834051:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02586a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x628:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.834057:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02586a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x628:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.834065:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.834067:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363fe56300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+731:2387:x1870153002399104:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 209us (646us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.834071:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025855 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x624:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.834074:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.834079:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35931 00010000:00010000:3.0:1783520935.834085:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025855 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x624:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.834089:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363fe55180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+731:2387:x1870153002399488:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 197us (633us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.834099:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e004e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+730:2387:x1870153002398848:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 166us (617us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.834102:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025855 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x624:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.834105:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35932 00010000:00010000:2.0:1783520935.834106:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02586a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x628:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.834115:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35933 00010000:00010000:3.0:1783520935.834125:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025855 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x624:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.834126:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02586a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x628:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.834143:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025871 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x627:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.834146:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02586a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x628:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.834149:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000441e0c6f/0xd5401cbabc0200b9 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x629:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0804 expref: 728 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.834154:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.834163:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025871 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x627:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.834171:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02586a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x628:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.834176:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513139880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993120128:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.834191:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025871 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x627:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.834196:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520935.834196:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc025878 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x629:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.834208:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520935.834208:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025871 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x627:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.834215:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025855 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x624:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.834215:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc025878 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x629:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.834231:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520935.834237:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.834244:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025855 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x624:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.834245:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc025878 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x629:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.834252:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350d165f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+272:3626:x1870152993118592:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5342us (5932us total) trans 4294975837 rc 0/0 00010000:00010000:2.0:1783520935.834263:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025855 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x624:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.834269:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc025878 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x629:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.834272:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44337 00010000:00010000:2.0:1783520935.834275:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025855 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x624:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.834282:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc025878 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x629:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.834284:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025855 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x624:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.834306:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to regular blp list, 3 blwis in pool 00010000:00010000:2.0:1783520935.834312:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:2.0:1783520935.834315:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02586a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x628:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.834317:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00000004:00080000:3.0:1783520935.834320:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350d167100/2, rc 0, transno 4294975837 00010000:00010000:2.0:1783520935.834324:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.834325:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025871 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x627:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.834327:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350544e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+272:3626:x1870152993119360:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4517us (4882us total) trans 4294975838 rc 0/0 00010000:00010000:2.0:1783520935.834330:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02586a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x628:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.834333:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350d167100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993118592:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.834345:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44341 00010000:00010000:1.0:1783520935.834348:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.834354:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025871 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x627:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.834381:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025871 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x627:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.834382:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02586a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x628:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.834386:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350544ca80/2, rc 0, transno 4294975838 00000100:00100000:3.0:1783520935.834397:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350544ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993119360:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.834397:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.834406:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc025878 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x629:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.834411:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025871 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x627:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.834413:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02586a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x628:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.834419:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9435122dd400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.834426:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc025878 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x629:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.834439:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025871 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x627:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.834440:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02586a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x628:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.834442:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc025878 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x629:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.834475:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc025878 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x629:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.834476:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.834480:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.834481:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.834482:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350544ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993120640:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.834483:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.834486:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.834487:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc025878 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x629:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.834490:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.834493:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.834519:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.834522:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.834525:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.834534:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.834537:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02584e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x623:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.834548:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02584e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x623:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.834557:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02584e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x623:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.834564:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:0.0:1783520935.834565:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02584e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x623:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.834574:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350544f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993120512:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.834589:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.834598:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520935.834601:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350544df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+273:3628:x1870152993119744:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4768us (4854us total) trans 4294975839 rc 0/0 00010000:00010000:2.0:1783520935.834605:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02584e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x623:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.834611:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44342 00000100:00100000:3.0:1783520935.834612:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.834622:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.834626:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02584e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x623:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.834634:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02584e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x623:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.834636:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993120640 00000100:00100000:1.0:1783520935.834643:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993120640 00010000:00010000:2.0:1783520935.834647:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02584e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x623:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.834653:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02584e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x623:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.834663:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350544fb80/2, rc 0, transno 4294975839 00010000:00010000:2.0:1783520935.834665:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.834669:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.834671:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.834675:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993120512 00000100:00100000:3.0:1783520935.834677:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350544fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993119744:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.834680:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993120512 00000100:00100000:0.0:1783520935.834704:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350d166a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+275:3626:x1870152993118464:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6388us (6583us total) trans 4294975841 rc 0/0 00000100:00100000:1.0:1783520935.834711:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44344 00000100:00100000:2.0:1783520935.834717:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350544d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+275:3626:x1870152993119488:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4987us (5119us total) trans 4294975840 rc 0/0 00000100:00100000:1.0:1783520935.834724:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516681880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+275:3626:x1870152993120640:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.834726:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44336 00000100:00100000:2.0:1783520935.834735:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44340 00000004:00080000:3.0:1783520935.834740:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350544f800/2, rc 0, transno 4294975840 00000100:00100000:3.0:1783520935.834750:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350544f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993119488:0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.834750:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1580 count 1 00000100:00100000:2.0:1783520935.834759:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44345 00000100:00100000:0.0:1783520935.834759:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516681180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993120768:0@lo:6:osp-syn-1-0.0 00000004:00080000:3.0:1783520935.834767:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350d166680/2, rc 0, transno 4294975841 00010000:00010000:1.0:1783520935.834767:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc025886 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x62c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.834773:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516682300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+273:3626:x1870152993120512:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.834777:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350d166680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993118464:0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.834791:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1579 count 1 00010000:00010000:1.0:1783520935.834791:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000853885c8/0xd5401cbabc0200d5 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x62c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f082e expref: 727 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.834792:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.834804:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x62c:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.834805:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02588d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x62b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.834813:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x62c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.834821:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc025886 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x62c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.834823:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001a5ac5bf/0xd5401cbabc0200ce lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x62b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0820 expref: 727 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.834823:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1783520935.834831:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1783520935.834836:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516682680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993120896:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.834845:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516683b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993121024:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.834847:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x62b:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.834853:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x62b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.834856:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02588d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x62b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.834863:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000853885c8/0xd5401cbabc0200d5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x62c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f082e expref: 727 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.834869:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.834878:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001a5ac5bf/0xd5401cbabc0200ce lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x62b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0820 expref: 727 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.834897:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000853885c8/0xd5401cbabc0200d5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x62c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f082e expref: 727 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.834898:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.834906:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001a5ac5bf/0xd5401cbabc0200ce lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x62b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0820 expref: 727 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.834911:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993120768 00000100:00100000:0.0:1783520935.834918:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993120768 00000100:00100000:3.0:1783520935.834924:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993120896 00000100:00100000:2.0:1783520935.834927:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5bb800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993121280:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.834927:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e005500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993121152:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.834929:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993120896 00000100:00100000:0.0:1783520935.834941:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993121024 00000100:00100000:0.0:1783520935.834946:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993121024 00000100:00100000:3.0:1783520935.834954:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44346 00000100:00100000:3.0:1783520935.834962:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+276:3627:x1870152993120768:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520935.834974:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1581 count 1 00000100:00100000:0.0:1783520935.834978:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44347 00000100:00100000:2.0:1783520935.834979:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676780 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.834980:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c369480 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.834985:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025894 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x62d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.834994:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+276:3627:x1870152993120896:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.834999:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000031b832c6/0xd5401cbabc0200dc lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x62d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f083c expref: 727 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.834999:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44348 00000100:00100000:2.0:1783520935.835010:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516682a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+276:3626:x1870152993121024:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.835012:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x62d:0x0].0x0 since=37s v=(0 0) 00002000:00080000:0.0:1783520935.835013:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1582 count 1 00010000:00010000:3.0:1783520935.835020:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x62d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.835025:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025894 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x62d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.835025:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1583 count 1 00010000:00010000:0.0:1783520935.835027:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02589b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x62e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.835031:0:12734:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000202a1452/0xd5401cbabc02289e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x626:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f8e expref: 727 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.835042:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0258a2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x62f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.835060:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000714a0b57/0xd5401cbabc022af8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x62e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f31e8 expref: 727 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.835065:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000031b832c6/0xd5401cbabc0200dc lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x62d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f083c expref: 726 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.835066:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351313b100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993119616:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.835068:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000015e0b25d/0xd5401cbabc020145 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x62f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0858 expref: 726 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.835085:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x62e:0x0].0x0 since=36s v=(0 0) 00010000:00010000:1.0:1783520935.835088:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025863 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x626:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.835092:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000031b832c6/0xd5401cbabc0200dc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x62d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f083c expref: 726 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.835092:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x62f:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.835095:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x62e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.835101:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x62f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.835105:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02589b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x62e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.835110:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0258a2 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x62f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.835117:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025863 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x626:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.835123:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516682d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993121408:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.835137:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025863 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x626:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.835146:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025863 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x626:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.835149:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000714a0b57/0xd5401cbabc022af8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x62e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31e8 expref: 726 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.835151:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967840 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.835151:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000015e0b25d/0xd5401cbabc020145 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x62f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0858 expref: 726 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.835173:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025863 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x626:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.835179:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000015e0b25d/0xd5401cbabc020145 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x62f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0858 expref: 726 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.835180:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000714a0b57/0xd5401cbabc022af8 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x62e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31e8 expref: 726 pid: 8410 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.835214:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9d180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993121536:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.835224:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b8380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993121664:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.835227:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.835238:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.835243:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025863 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x626:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.835250:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470900 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.835252:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9435122dcc00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520935.835253:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676a80 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.835261:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025863 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x626:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.835269:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025863 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x626:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.835283:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025863 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x626:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.835292:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.835296:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025863 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x626:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.835314:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002399232 00000100:00100000:2.0:1783520935.835318:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002399232 00010000:00010000:1.0:1783520935.835318:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.835329:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.835334:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.835337:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000fd9544ad/0xd5401cbabc0200ab lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x625:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07cc expref: 727 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.835380:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35934 00000100:00100000:2.0:1783520935.835390:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5b8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+727:2387:x1870153002399232:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.835400:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.835406:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.835408:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000fd9544ad/0xd5401cbabc0200ab lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x625:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07cc expref: 727 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.835427:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000fd9544ad/0xd5401cbabc0200ab lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x625:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07cc expref: 727 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.835428:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002399616 00000100:00100000:1.0:1783520935.835434:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002399616 00010000:00010000:2.0:1783520935.835446:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000fd9544ad/0xd5401cbabc0200ab lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x625:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f07cc expref: 728 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.835458:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x625:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.835458:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0200ea lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x62a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0812 expref: 727 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.835466:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x625:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.835476:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02585c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x625:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.835483:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350544d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+277:3627:x1870152993119232:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6058us (6211us total) trans 4294975842 rc 0/0 00000100:00100000:0.0:1783520935.835512:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44339 00010000:00010000:2.0:1783520935.835514:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02585c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x625:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.835517:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35935 00000100:00100000:1.0:1783520935.835531:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e007480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+727:2387:x1870153002399616:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.835541:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.835544:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.835549:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0200ea lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x62a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0812 expref: 727 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.835555:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350544d880/2, rc 0, transno 4294975842 00010000:00010000:1.0:1783520935.835566:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0200ea lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x62a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0812 expref: 727 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.835568:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5b8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+727:2387:x1870153002399232:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 178us (274us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.835575:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350544d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993119232:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.835587:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35934 00010000:00010000:1.0:1783520935.835597:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0200ea lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x62a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0812 expref: 727 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.835603:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02585c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x625:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.835611:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02585c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x625:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.835613:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x62a:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.835617:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516680380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993121792:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.835620:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02585c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x625:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.835621:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x62a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.835632:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02587f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x62a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.835632:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02585c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x625:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.835658:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02587f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x62a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.835664:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.835665:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.835675:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.835682:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02585c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x625:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.835688:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.835691:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.835696:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02585c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x625:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.835705:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02585c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x625:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.835712:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e007480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+725:2387:x1870153002399616:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 183us (307us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.835723:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35935 00010000:00010000:2.0:1783520935.835726:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02585c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x625:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.835736:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02585c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x625:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.835754:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993121792 00010000:00010000:2.0:1783520935.835759:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.835760:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993121792 00010000:00010000:2.0:1783520935.835763:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.835766:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638997800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+277:3627:x1870152993118848:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6803us (6955us total) trans 4294975843 rc 0/0 00010000:00010000:2.0:1783520935.835767:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.835776:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44338 00010000:00010000:2.0:1783520935.835781:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02587f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x62a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.835791:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02587f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x62a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.835798:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44349 00010000:00010000:2.0:1783520935.835803:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02587f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x62a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.835810:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634581850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+277:3626:x1870152993121792:12345-0@lo:6:osp-syn-1-0.0 00000004:00080000:0.0:1783520935.835814:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350d166d80/2, rc 0, transno 4294975843 00010000:00010000:2.0:1783520935.835817:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02587f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x62a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.835829:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1584 count 1 00000100:00100000:0.0:1783520935.835832:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350d166d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993118848:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.835835:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.835845:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0258a9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x630:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.835853:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.835859:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02587f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x62a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.835860:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000eb723682/0xd5401cbabc0200f1 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x630:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0866 expref: 724 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.835868:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:0.0:1783520935.835875:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.835879:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x630:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.835880:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02587f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x62a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.835882:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516683480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993121920:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.835889:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x630:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.835899:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0258a9 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x630:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.835901:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350544ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+278:3628:x1870152993120256:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5544us (5717us total) trans 4294975844 rc 0/0 00010000:00010000:0.0:1783520935.835916:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02587f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x62a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.835918:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44343 00000100:00100000:3.0:1783520935.835924:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.835933:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02587f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x62a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.835937:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993121920 00010000:00010000:1.0:1783520935.835939:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000eb723682/0xd5401cbabc0200f1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x630:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0866 expref: 724 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.835941:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993121920 00010000:00010000:0.0:1783520935.835944:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02587f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x62a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.835959:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350544dc00/2, rc 0, transno 4294975844 00000100:00100000:2.0:1783520935.835964:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44350 00010000:00010000:1.0:1783520935.835966:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000eb723682/0xd5401cbabc0200f1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x630:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0866 expref: 724 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.835969:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.835973:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.835974:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516680e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+278:3628:x1870152993121920:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.835975:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350544dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993120256:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.835976:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00002000:00080000:2.0:1783520935.835990:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1585 count 1 00000100:00100000:1.0:1783520935.835993:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e007800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993122048:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.836003:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0258b0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x631:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.836017:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d5b8c0a0/0xd5401cbabc0200f8 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x631:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0874 expref: 724 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.836031:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943634572300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993122176:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.836031:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x631:0x0].0x0 since=37s v=(0 0) 00000100:00100000:1.0:1783520935.836033:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6cc0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.836039:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x631:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.836044:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0258b0 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x631:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.836065:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.836068:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d5b8c0a0/0xd5401cbabc0200f8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x631:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0874 expref: 724 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.836079:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d5b8c0a0/0xd5401cbabc0200f8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x631:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0874 expref: 724 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.836090:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b9180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993122304:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.836102:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993122176 00000100:00100000:0.0:1783520935.836107:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993122176 00000100:00100000:2.0:1783520935.836113:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676840 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.836141:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44351 00000100:00100000:0.0:1783520935.836152:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634571c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+279:3628:x1870152993122176:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.836167:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1586 count 1 00010000:00010000:0.0:1783520935.836181:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0258b7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x632:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.836193:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f1e495b5/0xd5401cbabc0200ff lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x632:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0882 expref: 724 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.836201:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x632:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.836205:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x632:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.836210:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0258b7 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x632:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.836235:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f1e495b5/0xd5401cbabc0200ff lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x632:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0882 expref: 724 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.836247:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f1e495b5/0xd5401cbabc0200ff lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x632:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0882 expref: 724 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.836259:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9ce00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993122432:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.836291:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.839612:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9ce00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993122432:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.839647:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0258b7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x632:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.839667:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e005500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993121152:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.839703:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9d180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993121536:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.839719:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02589b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x62e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.839728:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.839753:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e007800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993122048:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.839773:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc025886 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x62c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.839775:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0258a9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x630:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.839783:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b9180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993122304:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.839803:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0258b0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x631:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.839817:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002399872 00000100:00100000:2.0:1783520935.839822:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5bb800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993121280:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.839825:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002399872 00010000:00010000:2.0:1783520935.839843:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02588d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x62b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.839849:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b8380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993121664:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.839861:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f1e495b5/0xd5401cbabc0200ff lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x632:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0882 expref: 725 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.839866:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0258a2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x62f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.839880:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516682d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993121408:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.839897:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025894 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x62d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.839905:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35936 00000100:00100000:0.0:1783520935.839915:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9ea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+725:2388:x1870153002399872:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.839927:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.839931:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f1e495b5/0xd5401cbabc0200ff lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x632:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0882 expref: 725 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.839940:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f1e495b5/0xd5401cbabc0200ff lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x632:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0882 expref: 725 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.839958:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f1e495b5/0xd5401cbabc0200ff lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x632:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0882 expref: 725 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.839970:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x632:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.839973:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x632:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.839979:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0258b7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x632:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.839987:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0258b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x632:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.840016:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.840032:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0258b7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x632:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.840042:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0258b7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x632:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.840052:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0258b7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x632:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.840061:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0258b7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x632:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.840068:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9ea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+724:2388:x1870153002399872:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 154us (346us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.840083:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35936 00010000:00010000:1.0:1783520935.840100:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.840108:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.840117:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0258b7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x632:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.840126:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.840131:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0258b7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x632:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.840139:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0258b7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x632:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.840152:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0258b7 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x632:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.840157:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0258b7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x632:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.840179:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.840183:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.840186:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.840187:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.840215:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002399744 00000100:00100000:0.0:1783520935.840219:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002399744 00010000:00010000:0.0:1783520935.840241:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000853885c8/0xd5401cbabc0200d5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x62c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f082e expref: 724 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.840273:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35937 00000100:00100000:1.0:1783520935.840279:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634571c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+280:3628:x1870152993122176:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4130us (4213us total) trans 4294975845 rc 0/0 00000100:00100000:0.0:1783520935.840285:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638584050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+724:2387:x1870153002399744:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.840292:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44351 00010000:00010000:0.0:1783520935.840297:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.840304:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000853885c8/0xd5401cbabc0200d5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x62c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f082e expref: 724 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.840318:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000853885c8/0xd5401cbabc0200d5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x62c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f082e expref: 724 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.840329:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000004:00080000:3.0:1783520935.840330:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943634572300/2, rc 0, transno 4294975845 00010000:00010000:0.0:1783520935.840339:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000853885c8/0xd5401cbabc0200d5 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x62c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f082e expref: 724 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.840347:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943634572300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993122176:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.840353:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x62c:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.840381:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002400128 00010000:00010000:0.0:1783520935.840381:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x62c:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.840388:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002400128 00010000:00010000:0.0:1783520935.840390:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc025886 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x62c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.840400:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc025886 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x62c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.840410:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000714a0b57/0xd5401cbabc022af8 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x62e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31e8 expref: 724 pid: 8410 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.840422:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.840441:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35938 00000100:00100000:2.0:1783520935.840451:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5bb800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+724:2388:x1870153002400128:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.840455:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638584050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+724:2387:x1870153002399744:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 172us (271us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.840463:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.840467:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35937 00010000:00010000:2.0:1783520935.840473:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000714a0b57/0xd5401cbabc022af8 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x62e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31e8 expref: 724 pid: 8410 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.840488:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000714a0b57/0xd5401cbabc022af8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x62e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31e8 expref: 723 pid: 8410 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.840489:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.840510:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943634573800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993122560:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.840523:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000714a0b57/0xd5401cbabc022af8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x62e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31e8 expref: 723 pid: 8410 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.840532:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x62e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.840535:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x62e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.840539:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02589b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x62e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.840539:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002400000 00000100:00100000:0.0:1783520935.840545:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002400000 00010000:00010000:2.0:1783520935.840547:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02589b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x62e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.840561:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.840566:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.840566:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001a5ac5bf/0xd5401cbabc0200ce lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x62b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0820 expref: 723 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.840594:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5bb800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+723:2388:x1870153002400128:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 143us (268us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.840595:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.840598:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35939 00000100:00100000:0.0:1783520935.840612:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e007800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+723:2387:x1870153002400000:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.840624:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520935.840625:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35938 00010000:00010000:0.0:1783520935.840632:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001a5ac5bf/0xd5401cbabc0200ce lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x62b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0820 expref: 723 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.840645:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002400512 00010000:00010000:0.0:1783520935.840646:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001a5ac5bf/0xd5401cbabc0200ce lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x62b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0820 expref: 722 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.840650:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002400512 00010000:00010000:0.0:1783520935.840666:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001a5ac5bf/0xd5401cbabc0200ce lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x62b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0820 expref: 723 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.840670:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d5b8c0a0/0xd5401cbabc0200f8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x631:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0874 expref: 723 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.840678:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.840691:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x62b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.840696:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x62b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.840701:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02588d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x62b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.840710:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02588d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x62b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.840714:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002400384 00000100:00100000:1.0:1783520935.840719:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002400384 00000100:00100000:2.0:1783520935.840720:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35940 00000100:00100000:2.0:1783520935.840732:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e007b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+723:2388:x1870153002400512:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.840738:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.840741:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000015e0b25d/0xd5401cbabc020145 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x62f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0858 expref: 723 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.840743:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.840753:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d5b8c0a0/0xd5401cbabc0200f8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x631:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0874 expref: 723 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.840765:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e007800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+723:2387:x1870153002400000:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 154us (278us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.840769:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35941 00010000:00010000:2.0:1783520935.840770:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d5b8c0a0/0xd5401cbabc0200f8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x631:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0874 expref: 723 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.840780:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35939 00000100:00100000:1.0:1783520935.840784:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e004e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+723:2389:x1870153002400384:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.840795:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.840797:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d5b8c0a0/0xd5401cbabc0200f8 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x631:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0874 expref: 722 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.840803:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000015e0b25d/0xd5401cbabc020145 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x62f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0858 expref: 722 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.840814:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993122560 00010000:00010000:2.0:1783520935.840816:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x631:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.840820:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000015e0b25d/0xd5401cbabc020145 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x62f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0858 expref: 721 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.840820:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993122560 00010000:00010000:2.0:1783520935.840821:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x631:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.840831:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0258b0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x631:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.840841:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0258b0 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x631:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.840844:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000015e0b25d/0xd5401cbabc020145 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x62f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0858 expref: 721 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.840857:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44352 00000100:00100000:0.0:1783520935.840870:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634573b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+280:3625:x1870152993122560:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.840875:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x62f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.840878:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.840880:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x62f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.840886:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0258a2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x62f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.840895:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0258a2 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x62f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.840895:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1587 count 1 00000100:00100000:2.0:1783520935.840909:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e007b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+720:2388:x1870153002400512:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 177us (316us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.840917:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0258be lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x633:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.840920:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35940 00010000:00010000:1.0:1783520935.840921:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.840937:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc020122 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x633:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0890 expref: 719 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.840945:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x633:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.840947:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02588d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x62b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.840949:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e004e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+719:2389:x1870153002400384:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 168us (273us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.840954:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x633:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.840964:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0258be lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x633:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.840967:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02588d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x62b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.840969:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35941 00010000:00010000:2.0:1783520935.840989:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02588d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x62b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.840997:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02588d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x62b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.841003:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc020122 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x633:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0890 expref: 718 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.841008:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02589b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x62e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.841022:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.841025:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02589b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x62e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.841028:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc020122 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x633:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0890 expref: 718 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.841044:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02589b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x62e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.841052:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9ea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993122688:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.841055:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02589b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x62e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.841070:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.841075:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02588d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x62b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.841083:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.841087:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02588d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x62b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.841097:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02588d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x62b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.841111:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02588d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x62b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.841112:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.841122:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02588d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x62b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.841131:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0258b0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x631:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.841145:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.841146:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516682300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+281:3626:x1870152993120512:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6374us (6534us total) trans 4294975846 rc 0/0 00010000:00010000:1.0:1783520935.841149:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.841151:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0258b0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x631:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.841154:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350544f100/2, rc 0, transno 4294975846 00010000:00010000:1.0:1783520935.841154:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.841161:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44345 00000100:00100000:3.0:1783520935.841170:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350544f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993120512:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.841171:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0258b0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x631:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.841182:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0258b0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x631:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.841188:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc025886 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x62c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.841198:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc025886 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x62c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.841203:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.841213:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc025886 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x62c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.841222:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943634570380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993122816:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.841233:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc025886 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x62c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.841235:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520935.841253:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520935.841258:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0258b0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x631:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.841259:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.841269:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.841273:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0258b0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x631:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.841278:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0258b0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x631:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.841289:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0258b0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x631:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.841294:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0258b0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x631:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.841301:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516680e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+282:3628:x1870152993121920:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5328us (5379us total) trans 4294975847 rc 0/0 00000100:00100000:0.0:1783520935.841310:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44350 00010000:00010000:2.0:1783520935.841311:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.841315:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02589b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x62e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.841317:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516683480/2, rc 0, transno 4294975847 00010000:00010000:2.0:1783520935.841325:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.841329:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02589b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x62e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.841330:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993122816 00000100:00100000:3.0:1783520935.841334:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516683480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993121920:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.841337:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02589b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x62e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.841337:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993122816 00000100:00100000:1.0:1783520935.841340:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+281:3627:x1870152993120896:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6349us (6470us total) trans 4294975848 rc 0/0 00000100:00100000:3.0:1783520935.841409:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1783520935.841419:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44347 00000100:00100000:3.0:1783520935.841420:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993122944:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.841435:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02589b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x62e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.841441:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44353 00010000:00010000:2.0:1783520935.841450:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02589b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x62e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.841457:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634571f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+281:3626:x1870152993122816:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.841460:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00080000:0.0:1783520935.841477:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1588 count 1 00010000:00010000:2.0:1783520935.841478:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.841478:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.841482:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.841486:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.841494:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0258c5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x634:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.841513:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.841521:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000473b5072/0xd5401cbabc020106 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x634:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f089e expref: 718 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.841525:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000004:00080000:3.0:1783520935.841527:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516682680/2, rc 0, transno 4294975848 00010000:00010000:2.0:1783520935.841533:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc025886 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x62c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.841535:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x634:0x0].0x0 since=37s v=(0 0) 00000100:00100000:3.0:1783520935.841542:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516682680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993120896:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.841544:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x634:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.841552:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.841555:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0258c5 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x634:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.841557:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9435122dca00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.841564:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc025886 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x62c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.841575:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc025886 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x62c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.841588:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000473b5072/0xd5401cbabc020106 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x634:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f089e expref: 718 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.841598:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc025886 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x62c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.841603:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000473b5072/0xd5401cbabc020106 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x634:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f089e expref: 718 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.841606:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993123072:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.841617:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc025886 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x62c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.841628:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9d500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993123200:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.841640:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.841644:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.841645:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993122944 00010000:00010000:2.0:1783520935.841648:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.841651:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993122944 00000100:00100000:3.0:1783520935.841652:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.841683:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993123072 00000100:00100000:0.0:1783520935.841686:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470f00 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.841688:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993123072 00010000:00010000:0.0:1783520935.841705:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.841715:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44354 00000100:00100000:0.0:1783520935.841720:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002400256 00000100:00100000:3.0:1783520935.841724:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44355 00000100:00100000:1.0:1783520935.841726:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032f0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+284:3628:x1870152993122944:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.841727:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002400256 00000100:00100000:3.0:1783520935.841738:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+284:3628:x1870152993123072:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.841743:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1589 count 1 00010000:00010000:0.0:1783520935.841750:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000031b832c6/0xd5401cbabc0200dc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x62d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f083c expref: 719 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.841759:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000007d2c1b3/0xd5401cbabc0258cc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x635:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.841760:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516681880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+284:3626:x1870152993120640:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7037us (7224us total) trans 4294975849 rc 0/0 00000004:00080000:3.0:1783520935.841771:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350544ce00/2, rc 0, transno 4294975849 00000100:00100000:0.0:1783520935.841775:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002400640 00000100:00100000:2.0:1783520935.841780:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44344 00000100:00100000:0.0:1783520935.841781:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002400640 00000100:00100000:3.0:1783520935.841786:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350544ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993120640:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.841787:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc02010d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x635:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f08ac expref: 719 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.841798:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x635:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.841800:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000eb723682/0xd5401cbabc0200f1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x630:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0866 expref: 720 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.841804:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x635:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.841813:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000007d2c1b3/0xd5401cbabc0258cc lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x635:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.841820:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35942 00000100:00100000:2.0:1783520935.841834:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e007480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+720:2387:x1870153002400256:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520935.841838:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993123328:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.841845:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.841850:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35943 00010000:00010000:2.0:1783520935.841853:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000031b832c6/0xd5401cbabc0200dc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x62d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f083c expref: 720 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.841863:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e005500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+720:2387:x1870153002400640:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.841864:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000031b832c6/0xd5401cbabc0200dc lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x62d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f083c expref: 720 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.841869:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.841875:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc02010d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x635:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08ac expref: 720 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.841881:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.841891:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000031b832c6/0xd5401cbabc0200dc lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x62d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f083c expref: 720 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.841892:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000eb723682/0xd5401cbabc0200f1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x630:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0866 expref: 720 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.841896:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc02010d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x635:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08ac expref: 720 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520935.841906:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1590 count 1 00010000:00010000:3.0:1783520935.841920:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0258d3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x636:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.841924:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000eb723682/0xd5401cbabc0200f1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x630:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0866 expref: 720 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.841930:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d322300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993123456:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.841934:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x62d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.841944:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x62d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.841948:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006b05df37/0xd5401cbabc0229e7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x636:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f30d7 expref: 719 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.841949:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025894 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x62d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.841957:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000eb723682/0xd5401cbabc0200f1 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x630:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0866 expref: 719 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.841965:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6cc0 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.841968:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x636:0x0].0x0 since=36s v=(0 0) 00010000:00010000:3.0:1783520935.841976:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x636:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.841976:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025894 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x62d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.841983:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0258d3 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x636:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.841985:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x630:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.841993:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x630:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.842002:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0258a9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x630:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.842006:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.842014:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0258a9 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x630:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.842017:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006b05df37/0xd5401cbabc0229e7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x636:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30d7 expref: 718 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.842024:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025894 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x62d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.842028:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0258a2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x62f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.842044:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.842047:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006b05df37/0xd5401cbabc0229e7 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x636:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30d7 expref: 718 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.842054:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025894 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x62d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.842054:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0258a2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x62f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.842072:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e005500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+718:2387:x1870153002400640:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 211us (523us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.842085:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025894 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x62d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.842085:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0258a2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x62f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.842092:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35943 00000100:00100000:3.0:1783520935.842093:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032f2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993123584:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.842114:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025894 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x62d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.842115:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0258a2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x62f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.842123:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993123328 00000100:00100000:0.0:1783520935.842129:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993123328 00010000:00010000:1.0:1783520935.842145:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520935.842151:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520935.842156:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00000100:00100000:0.0:1783520935.842157:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44356 00010000:00010000:1.0:1783520935.842163:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0258a2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x62f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.842170:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032f3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+284:3625:x1870152993123328:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.842172:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.842179:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0258a2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x62f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.842187:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0258a2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x62f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.842187:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1591 count 1 00000100:00100000:3.0:1783520935.842195:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967840 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.842203:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0258a2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x62f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.842205:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0258da lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x637:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.842221:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0258a2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x62f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.842228:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c3be515a/0xd5401cbabc02011b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x637:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f08c8 expref: 717 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.842238:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x637:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.842241:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x637:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.842245:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0258da lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x637:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.842247:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.842254:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025894 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x62d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.842255:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+285:3627:x1870152993120768:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7293us (7461us total) trans 4294975850 rc 0/0 00000004:00080000:3.0:1783520935.842260:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516681180/2, rc 0, transno 4294975850 00000100:00100000:2.0:1783520935.842273:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44346 00010000:00010000:0.0:1783520935.842273:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c3be515a/0xd5401cbabc02011b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x637:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08c8 expref: 717 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.842274:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4a00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:3.0:1783520935.842277:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516681180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993120768:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.842282:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025894 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x62d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.842295:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.842302:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c3be515a/0xd5401cbabc02011b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x637:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08c8 expref: 717 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.842303:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025894 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x62d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.842319:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e007480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+717:2387:x1870153002400256:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 487us (841us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.842327:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:1.0:1783520935.842327:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025894 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x62d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.842328:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35942 00000100:00100000:0.0:1783520935.842332:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9df80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993123712:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.842338:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993123840:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.842349:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025894 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x62d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.842391:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.842395:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.842396:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.842400:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.842411:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470900 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.842424:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993123840 00000100:00100000:1.0:1783520935.842430:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993123840 00010000:00010000:0.0:1783520935.842435:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0258a9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x630:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.842445:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0258a9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x630:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.842454:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0258a9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x630:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.842463:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0258a9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x630:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.842477:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44357 00010000:00010000:0.0:1783520935.842479:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.842491:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032f1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+285:3625:x1870152993123840:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.842520:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.842525:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0258a9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x630:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.842528:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1592 count 1 00010000:00010000:2.0:1783520935.842532:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9435122dde00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.842536:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0258a9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x630:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.842543:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0258a9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x630:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.842550:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0258e1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x638:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.842553:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0258a9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x630:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.842570:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0258a9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x630:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.842582:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc020129 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x638:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f08d6 expref: 716 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.842588:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.842589:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634581850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+286:3626:x1870152993121792:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6778us (6922us total) trans 4294975851 rc 0/0 00010000:00010000:2.0:1783520935.842591:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520935.842594:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516680380/2, rc 0, transno 4294975851 00010000:00010000:2.0:1783520935.842595:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.842605:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44349 00010000:00010000:1.0:1783520935.842607:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x638:0x0].0x0 since=37s v=(0 0) 00000100:00100000:3.0:1783520935.842610:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516680380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993121792:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.842614:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x638:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.842621:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0258e1 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x638:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.842656:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc020129 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x638:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08d6 expref: 716 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.842669:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc020129 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x638:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08d6 expref: 716 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.842671:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993123968:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.842693:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d321c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993124096:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.842710:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.842738:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993123968 00000100:00100000:2.0:1783520935.842744:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993123968 00000100:00100000:1.0:1783520935.842747:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435131fce40 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.842753:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516682a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+287:3626:x1870152993121024:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7743us (7857us total) trans 4294975852 rc 0/0 00000100:00100000:0.0:1783520935.842764:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44348 00000004:00080000:3.0:1783520935.842769:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516683b80/2, rc 0, transno 4294975852 00000100:00100000:2.0:1783520935.842781:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44358 00000100:00100000:2.0:1783520935.842791:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506228a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+286:3627:x1870152993123968:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.842807:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1593 count 1 00010000:00010000:2.0:1783520935.842820:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0258e8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x639:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.842832:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009fcd7190/0xd5401cbabc020130 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x639:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f08e4 expref: 716 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.842840:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x639:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.842844:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x639:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.842847:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0258e8 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x639:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.842871:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009fcd7190/0xd5401cbabc020130 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x639:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08e4 expref: 716 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.842880:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516683b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993121024:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.842883:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009fcd7190/0xd5401cbabc020130 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x639:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08e4 expref: 716 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.842896:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b9f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993124224:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.842933:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435126766c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.842935:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515776a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993124352:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.842971:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.842991:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993124352 00000100:00100000:2.0:1783520935.842996:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993124352 00000100:00100000:2.0:1783520935.843027:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44359 00000100:00100000:2.0:1783520935.843036:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+287:3628:x1870152993124352:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.843049:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1594 count 1 00010000:00010000:2.0:1783520935.843062:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0258ef lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x63a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.843074:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005e2d2ef9/0xd5401cbabc020137 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x63a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f08f2 expref: 716 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.843082:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x63a:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.843086:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x63a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.843090:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0258ef lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x63a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.843115:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005e2d2ef9/0xd5401cbabc020137 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x63a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08f2 expref: 716 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.843125:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005e2d2ef9/0xd5401cbabc020137 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x63a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08f2 expref: 716 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.843135:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5baa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993124480:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.843181:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676d80 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.844675:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9ea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993122688:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.844698:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0258be lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x633:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.844910:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9d500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993123200:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.844931:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0258c5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x634:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.846625:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d322300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993123456:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.846627:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.846657:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000007d2c1b3/0xd5401cbabc0258cc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x635:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.846699:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002400768 00000100:00100000:2.0:1783520935.846706:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002400768 00000100:00100000:0.0:1783520935.846717:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.846748:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc020122 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x633:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0890 expref: 717 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.846756:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002400896 00000100:00100000:1.0:1783520935.846762:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002400896 00010000:00010000:1.0:1783520935.846783:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000473b5072/0xd5401cbabc020106 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x634:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f089e expref: 718 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.846790:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.846809:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b9f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993124224:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.846827:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.846831:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0258e8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x639:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.846839:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002401280 00000100:00100000:1.0:1783520935.846844:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002401280 00000100:00100000:0.0:1783520935.846856:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.846861:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002401152 00010000:00010000:1.0:1783520935.846861:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006b05df37/0xd5401cbabc0229e7 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x636:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30d7 expref: 719 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.846866:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002401152 00000100:00100000:1.0:1783520935.846882:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002401408 00000100:00100000:1.0:1783520935.846885:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002401408 00010000:00010000:2.0:1783520935.846889:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c3be515a/0xd5401cbabc02011b lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x637:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08c8 expref: 720 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.846894:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.846899:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc020129 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x638:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08d6 expref: 721 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.846916:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002401024 00000100:00100000:2.0:1783520935.846920:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002401024 00000100:00100000:1.0:1783520935.846922:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35944 00000100:00100000:1.0:1783520935.846933:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638586050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+722:2387:x1870153002400768:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.846940:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc02010d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x635:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08ac expref: 722 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.846951:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.846958:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc020122 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x633:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0890 expref: 722 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.846962:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35945 00000100:00100000:2.0:1783520935.846974:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9d500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+722:2387:x1870153002400896:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.846979:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc020122 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x633:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0890 expref: 722 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.846984:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.846992:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000473b5072/0xd5401cbabc020106 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x634:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f089e expref: 722 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.847002:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.847012:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000473b5072/0xd5401cbabc020106 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x634:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f089e expref: 722 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.847020:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc020122 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x633:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0890 expref: 722 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.847028:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002401536 00010000:00010000:2.0:1783520935.847032:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000473b5072/0xd5401cbabc020106 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x634:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f089e expref: 721 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.847034:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002401536 00010000:00010000:1.0:1783520935.847037:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x633:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.847046:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x633:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.847054:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x634:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.847056:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0258be lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x633:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.847058:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x634:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.847058:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009fcd7190/0xd5401cbabc020130 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x639:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08e4 expref: 721 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.847070:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0258c5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x634:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.847085:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0258be lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x633:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.847088:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0258c5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x634:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.847099:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35946 00000100:00100000:0.0:1783520935.847114:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9ea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+721:2390:x1870153002401280:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.847125:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.847132:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.847132:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.847133:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006b05df37/0xd5401cbabc0229e7 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x636:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30d7 expref: 721 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.847134:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032f2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993123584:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.847154:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006b05df37/0xd5401cbabc0229e7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x636:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30d7 expref: 721 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.847167:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0258d3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x636:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.847175:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006b05df37/0xd5401cbabc0229e7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x636:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30d7 expref: 721 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.847190:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638586050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+720:2387:x1870153002400768:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 257us (571us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.847193:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x636:0x0].0x0 since=0s v=(1 0) 00000100:00100000:2.0:1783520935.847194:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9d500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+720:2387:x1870153002400896:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 220us (474us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.847204:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x636:0x0].0x0 since=0s v=(1 0) 00000100:00100000:1.0:1783520935.847208:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35944 00010000:00010000:0.0:1783520935.847212:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0258d3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x636:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.847218:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35945 00010000:00010000:0.0:1783520935.847227:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0258d3 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x636:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.847245:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35947 00000100:00100000:2.0:1783520935.847254:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+718:2387:x1870153002401152:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.847257:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.847260:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0258d3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x636:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.847263:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.847271:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35948 00010000:00010000:2.0:1783520935.847274:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c3be515a/0xd5401cbabc02011b lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x637:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08c8 expref: 718 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.847280:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0258d3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x636:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.847285:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9ea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+718:2390:x1870153002401280:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 171us (495us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.847286:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+718:2387:x1870153002401408:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.847307:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c3be515a/0xd5401cbabc02011b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x637:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08c8 expref: 718 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.847307:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.847308:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35946 00010000:00010000:3.0:1783520935.847318:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0258d3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x636:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.847320:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc020129 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x638:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08d6 expref: 718 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.847328:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x637:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.847335:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35949 00010000:00010000:3.0:1783520935.847338:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0258d3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x636:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.847338:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x637:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.847348:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0258da lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x637:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.847349:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9ce00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+717:2387:x1870153002401024:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.847388:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.847391:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc020129 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x638:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08d6 expref: 717 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.847396:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0258da lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x637:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.847398:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc02010d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x635:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08ac expref: 717 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.847411:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.847427:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc02010d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x635:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08ac expref: 717 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.847430:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x638:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.847441:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x638:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.847452:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0258e1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x638:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.847457:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc02010d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x635:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08ac expref: 717 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.847460:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.847469:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0258e1 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x638:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.847477:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x635:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.847482:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x635:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.847485:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+716:2387:x1870153002401152:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 231us (658us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.847487:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.847489:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000007d2c1b3/0xd5401cbabc0258cc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x635:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.847514:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35947 00010000:00010000:0.0:1783520935.847516:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000007d2c1b3/0xd5401cbabc0258cc lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x635:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.847529:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+715:2387:x1870153002401408:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 244us (668us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.847536:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35950 00010000:00010000:0.0:1783520935.847546:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:2.0:1783520935.847552:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94361103b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+715:2387:x1870153002401536:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.847552:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35948 00010000:00010000:0.0:1783520935.847558:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0258d3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x636:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.847566:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.847569:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.847576:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009fcd7190/0xd5401cbabc020130 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x639:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08e4 expref: 714 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.847576:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0258d3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x636:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.847588:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+288:3628:x1870152993123072:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5851us (5935us total) trans 4294975853 rc 0/0 00010000:00010000:0.0:1783520935.847603:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0258d3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x636:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.847605:0:35276:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc020129 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x638:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08d6 expref: 714 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.847609:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44355 00010000:00010000:2.0:1783520935.847610:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009fcd7190/0xd5401cbabc020130 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x639:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08e4 expref: 714 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.847635:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0258d3 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x636:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.847637:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d321c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993124096:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.847644:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009fcd7190/0xd5401cbabc020130 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x639:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08e4 expref: 713 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.847656:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0258d3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x636:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.847659:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435032f1180/2, rc 0, transno 4294975853 00010000:00010000:1.0:1783520935.847663:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0258e1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x638:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.847664:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x639:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.847674:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993123072:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.847675:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x639:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.847687:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0258e8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x639:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.847689:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0258e1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x638:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.847694:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.847698:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.847702:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.847705:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0258e8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x639:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.847709:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0258e1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x638:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.847724:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.847733:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0258e1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x638:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.847743:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0258e1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x638:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.847747:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.847752:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9ce00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+712:2387:x1870153002401024:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 405us (860us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.847758:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.847762:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35949 00010000:00010000:2.0:1783520935.847764:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0258e8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x639:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.847774:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0258e8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x639:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.847784:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0258e8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x639:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.847793:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0258e8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x639:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.847804:0:20415:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c3be515a/0xd5401cbabc02011b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x637:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08c8 expref: 711 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.847816:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520935.847821:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1783520935.847826:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0258e1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x638:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.847829:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9df80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993123712:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.847844:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9435122dd200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.847847:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0258e1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x638:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.847854:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0258e1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x638:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.847858:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0258da lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x637:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.847863:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515776d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993124608:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.847865:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0258e1 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x638:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.847878:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0258da lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x637:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.847879:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000be59e5b9/0xd5401cbabc0258e1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x638:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.847900:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0258da lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x637:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.847908:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0258da lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x637:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.847918:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.847920:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0258da lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x637:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.847932:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506228a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+289:3627:x1870152993123968:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5142us (5219us total) trans 4294975855 rc 0/0 00010000:00010000:0.0:1783520935.847940:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520935.847942:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00000100:00100000:2.0:1783520935.847944:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44358 00010000:00010000:1.0:1783520935.847950:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0258e8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x639:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.847958:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.847962:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0258e8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x639:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.847965:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993124608 00000100:00100000:2.0:1783520935.847971:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993124608 00010000:00010000:1.0:1783520935.847975:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0258e8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x639:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.847990:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0258e8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x639:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.847999:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0258e8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x639:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.848008:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44360 00000100:00100000:2.0:1783520935.848019:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515777100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+290:3625:x1870152993124608:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.848021:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.848027:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0258da lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x637:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.848035:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.848040:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0258da lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x637:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.848047:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1595 count 1 00000100:00100000:0.0:1783520935.848049:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032f3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+290:3625:x1870152993123328:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5880us (6180us total) trans 4294975856 rc 0/0 00010000:00010000:1.0:1783520935.848052:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0258da lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x637:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.848056:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435032f2d80/2, rc 0, transno 4294975856 00000100:00100000:0.0:1783520935.848066:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44356 00000100:00100000:3.0:1783520935.848072:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993123328:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.848072:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0258f6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x63b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.848078:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0258da lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x637:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.848098:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0258da lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x637:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.848103:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02014c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x63b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0900 expref: 710 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.848108:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000007d2c1b3/0xd5401cbabc0258cc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x635:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.848126:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.848131:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515775500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993124736:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.848131:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x63b:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.848132:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.848132:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000007d2c1b3/0xd5401cbabc0258cc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x635:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.848138:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.848139:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x63b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.848150:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0258f6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x63b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.848154:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000007d2c1b3/0xd5401cbabc0258cc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x635:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.848169:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000007d2c1b3/0xd5401cbabc0258cc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x635:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.848170:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c5b9880/2, rc 0, transno 4294975855 00000100:00100000:3.0:1783520935.848174:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.848185:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02014c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x63b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0900 expref: 710 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.848190:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993123968:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.848193:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.848207:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02014c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x63b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0900 expref: 710 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.848209:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.848216:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000007d2c1b3/0xd5401cbabc0258cc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x635:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.848226:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b9f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993124864:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.848227:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9435122dda00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.848234:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000007d2c1b3/0xd5401cbabc0258cc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x635:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.848244:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000007d2c1b3/0xd5401cbabc0258cc lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x635:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.848249:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1783520935.848250:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032f1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+289:3625:x1870152993123840:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5761us (5852us total) trans 4294975854 rc 0/0 00000100:00100000:3.0:1783520935.848265:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515774a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993124992:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.848266:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44357 00010000:00010000:0.0:1783520935.848266:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000007d2c1b3/0xd5401cbabc0258cc lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x635:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.848281:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000007d2c1b3/0xd5401cbabc0258cc lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x635:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.848285:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676780 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.848291:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993124736 00000100:00100000:1.0:1783520935.848297:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993124736 00010000:00010000:2.0:1783520935.848303:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.848305:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.848306:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.848312:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.848313:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.848315:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.848326:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993124992 00000100:00100000:1.0:1783520935.848330:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993124992 00000100:00100000:0.0:1783520935.848338:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44361 00000100:00100000:2.0:1783520935.848339:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94361103b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+710:2387:x1870153002401536:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 788us (1336us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.848353:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35950 00000100:00100000:0.0:1783520935.848355:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515776680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+290:3626:x1870152993124736:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.848384:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44362 00000100:00100000:1.0:1783520935.848396:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515774380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+290:3628:x1870152993124992:12345-0@lo:6:osp-syn-1-0.0 00000004:00080000:3.0:1783520935.848401:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435032f1c00/2, rc 0, transno 4294975854 00002000:00080000:1.0:1783520935.848410:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1597 count 1 00000100:00100000:3.0:1783520935.848415:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993123840:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.848425:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0258fd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x63d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.848438:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b59b78ee/0xd5401cbabc02015a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x63d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f091c expref: 710 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.848439:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1596 count 1 00010000:00010000:1.0:1783520935.848447:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x63d:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.848450:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x63d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.848453:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0258fd lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x63d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.848458:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025904 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x63c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.848463:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0258c5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x634:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.848472:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b59b78ee/0xd5401cbabc02015a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x63d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f091c expref: 709 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.848474:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515774e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993125120:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.848478:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc020153 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x63c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f090e expref: 709 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.848484:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0258c5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x634:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.848511:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b59b78ee/0xd5401cbabc02015a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x63d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f091c expref: 709 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.848518:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x63c:0x0].0x0 since=37s v=(0 0) 00000100:00100000:3.0:1783520935.848523:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.848529:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x63c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.848539:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025904 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x63c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.848542:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0258c5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x634:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.848558:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993125120 00000100:00100000:3.0:1783520935.848564:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993125120 00000100:00100000:1.0:1783520935.848564:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d321c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993125248:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.848567:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0258c5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x634:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.848583:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc020153 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x63c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f090e expref: 709 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.848591:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.848597:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc020153 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x63c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f090e expref: 709 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.848602:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44363 00010000:00010000:1.0:1783520935.848603:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.848612:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0258c5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x634:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.848617:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515777480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+291:3628:x1870152993125120:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.848619:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9df80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993125376:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.848633:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbebe00 already unused, calling callback (ffffffffc0bf9e90) 00002000:00080000:3.0:1783520935.848637:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1598 count 1 00010000:00010000:1.0:1783520935.848637:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0258c5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x634:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.848651:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0258c5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x634:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.848653:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02590b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x63e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.848661:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470f00 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.848676:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000059081981/0xd5401cbabc0228d6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x63e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fc6 expref: 709 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.848678:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0258c5 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x634:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.848684:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x63e:0x0].0x0 since=36s v=(0 0) 00000100:00100000:2.0:1783520935.848689:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634571f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+292:3626:x1870152993122816:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7235us (7429us total) trans 4294975857 rc 0/0 00010000:00010000:3.0:1783520935.848691:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x63e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520935.848699:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02590b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x63e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.848701:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0258c5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x634:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.848708:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44353 00010000:00010000:3.0:1783520935.848728:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000059081981/0xd5401cbabc0228d6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x63e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fc6 expref: 709 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.848731:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.848737:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.848738:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000059081981/0xd5401cbabc0228d6 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x63e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fc6 expref: 709 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.848741:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.848751:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032f2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993125504:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.848752:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57d80 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.848772:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0258be lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x633:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.848780:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.848781:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967e40 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.848787:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032f0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+292:3628:x1870152993122944:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7061us (7329us total) trans 4294975858 rc 0/0 00010000:00010000:1.0:1783520935.848810:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0258be lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x633:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.848811:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44354 00000004:00080000:3.0:1783520935.848812:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943634570380/2, rc 0, transno 4294975857 00000100:00100000:3.0:1783520935.848828:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943634570380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993122816:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.848828:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0258be lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x633:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.848843:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0258be lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x633:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.848862:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002401664 00000100:00100000:1.0:1783520935.848867:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002401664 00000100:00100000:3.0:1783520935.848884:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:1.0:1783520935.848885:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005e2d2ef9/0xd5401cbabc020137 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x63a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08f2 expref: 710 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.848886:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.848896:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:3.0:1783520935.848898:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e471880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993125632:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.848903:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0258be lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x633:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.848912:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35951 00010000:00010000:2.0:1783520935.848918:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.848925:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0258be lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x633:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.848925:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943603c1ec50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+710:2388:x1870153002401664:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520935.848935:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.848941:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.848944:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0258be lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x633:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.848949:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005e2d2ef9/0xd5401cbabc020137 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x63a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08f2 expref: 710 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.848962:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0258be lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x633:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.848967:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005e2d2ef9/0xd5401cbabc020137 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x63a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08f2 expref: 710 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.848981:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943634573800/2, rc 0, transno 4294975859 00000100:00100000:0.0:1783520935.848983:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634573b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+292:3625:x1870152993122560:12345-0@lo:6:osp-syn-1-0.0 Request processed in 8114us (8422us total) trans 4294975859 rc 0/0 00000100:00100000:3.0:1783520935.848997:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943634573800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993122560:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.848999:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0258be lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x633:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.848999:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x63a:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.849003:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44352 00010000:00010000:1.0:1783520935.849011:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x63a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.849019:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0258ef lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.849028:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.849028:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0258ef lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x63a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.849028:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993125632 00010000:00010000:2.0:1783520935.849034:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.849035:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993125632 00010000:00010000:2.0:1783520935.849038:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.849050:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000004:00080000:3.0:1783520935.849060:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435032f1880/2, rc 0, transno 4294975858 00000100:00100000:0.0:1783520935.849070:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44364 00000100:00100000:3.0:1783520935.849074:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993122944:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.849083:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943603c1ec50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+710:2388:x1870153002401664:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 159us (305us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.849084:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e472680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+292:3626:x1870152993125632:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.849098:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35951 00002000:00080000:0.0:1783520935.849103:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1599 count 1 00010000:00010000:0.0:1783520935.849119:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025912 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x63f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.849133:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc020168 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x63f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0938 expref: 709 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.849142:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x63f:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.849146:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x63f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.849151:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025912 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x63f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.849158:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e470700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993125760:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.849175:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc020168 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x63f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0938 expref: 709 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.849187:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc020168 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x63f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0938 expref: 709 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.849190:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e472a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993125888:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.849197:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.849215:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9d500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993126016:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.849240:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.849252:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005e2d2ef9/0xd5401cbabc020137 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x63a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f08f2 expref: 709 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.849258:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993125760 00000100:00100000:3.0:1783520935.849263:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993125760 00000100:00100000:0.0:1783520935.849264:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470b40 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.849273:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5baa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993124480:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.849291:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993125888 00000100:00100000:3.0:1783520935.849294:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993125888 00010000:00010000:2.0:1783520935.849295:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0258ef lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.849299:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44365 00010000:00010000:2.0:1783520935.849310:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0258ef lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.849315:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e471c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+294:3626:x1870152993125760:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.849325:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0258ef lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.849333:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1600 count 1 00000100:00100000:0.0:1783520935.849334:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44366 00010000:00010000:2.0:1783520935.849335:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0258ef lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.849346:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0258ef lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x63a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.849349:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d322300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+294:3627:x1870152993125888:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.849350:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000016d99d15/0xd5401cbabc025919 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x640:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.849397:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1601 count 1 00010000:00010000:2.0:1783520935.849406:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.849408:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007b5733a3/0xd5401cbabc02016f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x640:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0946 expref: 708 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.849411:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025920 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x641:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.849417:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.849424:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0258ef lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x63a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.849427:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x640:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.849438:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x640:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.849442:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000141e6479/0xd5401cbabc020176 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x641:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0954 expref: 708 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.849443:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.849447:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000016d99d15/0xd5401cbabc025919 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x640:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.849450:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0258ef lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x63a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.849465:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x641:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.849477:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x641:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.849479:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0258ef lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x63a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.849488:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025920 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x641:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.849516:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007b5733a3/0xd5401cbabc02016f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x640:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0946 expref: 708 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.849530:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000141e6479/0xd5401cbabc020176 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x641:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0954 expref: 708 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.849534:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007b5733a3/0xd5401cbabc02016f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x640:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0946 expref: 708 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.849553:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000141e6479/0xd5401cbabc020176 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x641:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0954 expref: 708 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.849557:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d323800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993126144:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.849575:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9ea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993126272:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.849608:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435131fce40 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.849612:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.849633:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0258ef lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x63a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.849642:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0258ef lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x63a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.849659:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.849662:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.849664:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.849769:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+295:3628:x1870152993124352:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6734us (6797us total) trans 4294975860 rc 0/0 00000100:00100000:2.0:1783520935.849777:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44359 00000004:00080000:0.0:1783520935.849808:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515776a00/2, rc 0, transno 4294975860 00000100:00100000:0.0:1783520935.849826:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515776a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993124352:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.849914:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993126400:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.849952:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.849989:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993126400 00000100:00100000:2.0:1783520935.849994:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993126400 00000100:00100000:2.0:1783520935.850029:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44367 00000100:00100000:2.0:1783520935.850038:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+295:3628:x1870152993126400:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.850051:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1602 count 1 00010000:00010000:2.0:1783520935.850064:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025927 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x642:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.850076:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001f14d2b9/0xd5401cbabc02017d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x642:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0962 expref: 708 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.850083:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x642:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.850087:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x642:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.850091:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025927 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x642:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.850113:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001f14d2b9/0xd5401cbabc02017d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x642:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0962 expref: 708 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.850123:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001f14d2b9/0xd5401cbabc02017d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x642:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0962 expref: 708 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.850134:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b9c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993126528:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.850177:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676d80 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.852135:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b9f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993124864:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.852146:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d321c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993125248:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.852174:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0258f6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x63b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.852174:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0258fd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x63d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.852220:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032f2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993125504:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.852244:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9df80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993125376:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.852264:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025904 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x63c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.852290:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.852298:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02590b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x63e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.852311:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d323800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993126144:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.852323:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.852328:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000016d99d15/0xd5401cbabc025919 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x640:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.852383:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.852394:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9d500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993126016:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.852416:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025912 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x63f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.852424:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.852449:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002401792 00000100:00100000:2.0:1783520935.852457:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002401792 00000100:00100000:1.0:1783520935.852461:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002402048 00000100:00100000:0.0:1783520935.852466:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.852467:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002402048 00010000:00010000:1.0:1783520935.852492:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000059081981/0xd5401cbabc0228d6 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x63e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fc6 expref: 710 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.852492:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002402432 00010000:00010000:2.0:1783520935.852504:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b59b78ee/0xd5401cbabc02015a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x63d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f091c expref: 710 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.852512:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002402432 00010000:00010000:0.0:1783520935.852538:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000141e6479/0xd5401cbabc020176 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x641:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0954 expref: 711 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.852554:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002401920 00000100:00100000:2.0:1783520935.852559:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002401920 00000100:00100000:1.0:1783520935.852562:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002402176 00000100:00100000:1.0:1783520935.852567:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002402176 00000100:00100000:0.0:1783520935.852569:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35952 00010000:00010000:2.0:1783520935.852581:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc020153 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x63c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f090e expref: 713 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.852582:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+713:2389:x1870153002401792:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.852589:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc020168 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x63f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0938 expref: 713 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.852601:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.852610:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b59b78ee/0xd5401cbabc02015a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x63d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f091c expref: 713 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.852618:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35953 00010000:00010000:0.0:1783520935.852625:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b59b78ee/0xd5401cbabc02015a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x63d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f091c expref: 713 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.852632:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9ce00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+713:2389:x1870153002402048:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.852633:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35954 00010000:00010000:2.0:1783520935.852647:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.852648:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9df80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+713:2389:x1870153002402432:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.852656:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000059081981/0xd5401cbabc0228d6 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x63e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fc6 expref: 713 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.852658:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b59b78ee/0xd5401cbabc02015a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x63d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f091c expref: 713 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.852661:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.852671:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000141e6479/0xd5401cbabc020176 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x641:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0954 expref: 713 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.852685:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000059081981/0xd5401cbabc0228d6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x63e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fc6 expref: 712 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.852693:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x63d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.852694:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000141e6479/0xd5401cbabc020176 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x641:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0954 expref: 712 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.852701:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x63d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.852709:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000059081981/0xd5401cbabc0228d6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x63e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fc6 expref: 712 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.852713:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0258fd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.852719:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x641:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.852733:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0258fd lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x63d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.852734:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x63e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.852734:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x641:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.852743:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x63e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.852750:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025920 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x641:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.852752:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02590b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.852770:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.852771:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025920 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x641:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.852773:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02590b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x63e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.852800:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.852814:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+711:2389:x1870153002401792:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 235us (526us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.852819:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.852827:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35952 00010000:00010000:3.0:1783520935.852828:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02590b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.852828:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9df80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+711:2389:x1870153002402432:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 180us (468us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.852844:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35954 00010000:00010000:3.0:1783520935.852847:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02590b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.852848:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9ce00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+710:2389:x1870153002402048:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 218us (523us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.852855:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35955 00000100:00100000:1.0:1783520935.852867:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35956 00010000:00010000:3.0:1783520935.852869:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02590b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.852870:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508058700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+709:2389:x1870153002401920:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.852878:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35953 00000100:00100000:1.0:1783520935.852882:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350805b800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+709:2389:x1870153002402176:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.852887:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.852893:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02590b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x63e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.852898:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc020153 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x63c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f090e expref: 708 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.852901:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.852911:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc020168 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x63f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0938 expref: 708 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.852920:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.852923:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc020153 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x63c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f090e expref: 708 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.852928:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc020168 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x63f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0938 expref: 708 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.852932:0:35274:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000141e6479/0xd5401cbabc020176 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x641:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0954 expref: 708 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.852950:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc020153 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x63c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f090e expref: 708 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.852959:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9ea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993126272:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.852961:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc020168 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x63f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0938 expref: 707 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.852979:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025920 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x641:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.852983:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x63c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.852985:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x63f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.852992:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x63c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.852994:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x63f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.852998:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025920 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x641:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.853000:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025904 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.853002:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025912 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.853021:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025920 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x641:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.853023:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025904 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x63c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.853042:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025912 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x63f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.853048:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025920 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x641:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.853057:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515777480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+296:3628:x1870152993125120:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4440us (4533us total) trans 4294975861 rc 0/0 00000004:00080000:0.0:1783520935.853071:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515774e00/2, rc 0, transno 4294975861 00000100:00100000:3.0:1783520935.853077:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44363 00010000:00010000:2.0:1783520935.853087:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025920 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x641:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.853088:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.853091:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515774e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993125120:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.853116:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350805b800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+705:2389:x1870153002402176:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 237us (650us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.853117:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:1.0:1783520935.853127:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35956 00010000:00010000:1.0:1783520935.853154:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1783520935.853161:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02590b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x63e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.853170:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5a00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:3.0:1783520935.853173:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508058380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993126656:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.853174:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02590b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x63e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.853176:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.853185:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025920 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x641:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.853189:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02590b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x63e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.853200:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.853207:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025920 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x641:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.853211:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02590b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x63e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.853220:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.853222:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025920 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x641:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.853229:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02590b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x63e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.853235:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d322300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+296:3627:x1870152993125888:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3888us (3994us total) trans 4294975862 rc 0/0 00010000:00010000:0.0:1783520935.853243:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025920 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x641:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.853249:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44366 00010000:00010000:1.0:1783520935.853259:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.853263:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.853264:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025920 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x641:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.853267:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.853286:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993126656 00000100:00100000:3.0:1783520935.853292:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993126656 00010000:00010000:0.0:1783520935.853292:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.853297:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520935.853298:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e472a00/2, rc 0, transno 4294975862 00010000:00010000:0.0:1783520935.853303:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.853317:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e472a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993125888:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.853395:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350805bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993126784:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.853408:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44368 00000100:00100000:2.0:1783520935.853422:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e470000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+296:3625:x1870152993126656:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.853431:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00080000:2.0:1783520935.853442:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1603 count 1 00010000:00010000:0.0:1783520935.853449:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.853463:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc02592e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x643:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.853477:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508058700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+704:2389:x1870153002401920:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 610us (1057us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.853480:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc020184 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x643:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0970 expref: 704 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.853484:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.853487:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993126784 00000100:00100000:3.0:1783520935.853493:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993126784 00000100:00100000:0.0:1783520935.853508:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35955 00010000:00010000:2.0:1783520935.853515:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x643:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.853520:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x643:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.853525:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc02592e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x643:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.853530:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002402304 00000100:00100000:0.0:1783520935.853535:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002402304 00010000:00010000:2.0:1783520935.853551:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc020184 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x643:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0970 expref: 704 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.853556:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007b5733a3/0xd5401cbabc02016f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x640:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0946 expref: 704 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.853573:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc020184 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x643:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0970 expref: 704 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.853588:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b9f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993126912:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.853591:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35957 00000100:00100000:0.0:1783520935.853602:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d323800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+704:2388:x1870153002402304:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.853612:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.853616:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.853618:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44369 00010000:00010000:0.0:1783520935.853621:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007b5733a3/0xd5401cbabc02016f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x640:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0946 expref: 704 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.853632:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e473b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+297:3626:x1870152993126784:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.853638:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007b5733a3/0xd5401cbabc02016f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x640:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0946 expref: 704 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.853642:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676840 going to sleep for 11 seconds 00002000:00080000:3.0:1783520935.853648:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1604 count 1 00010000:00010000:0.0:1783520935.853660:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007b5733a3/0xd5401cbabc02016f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x640:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0946 expref: 704 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.853661:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002402560 00010000:00010000:3.0:1783520935.853665:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025935 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x644:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.853667:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002402560 00000100:00100000:1.0:1783520935.853668:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.853687:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x640:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.853688:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001f14d2b9/0xd5401cbabc02017d lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x642:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0962 expref: 704 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.853692:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc02018b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x644:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f097e expref: 704 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.853696:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x640:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.853709:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000016d99d15/0xd5401cbabc025919 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x640:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.853717:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x644:0x0].0x0 since=37s v=(0 0) 00000100:00100000:2.0:1783520935.853724:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002402688 00010000:00010000:3.0:1783520935.853726:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x644:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.853726:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000016d99d15/0xd5401cbabc025919 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x640:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.853729:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002402688 00010000:00010000:3.0:1783520935.853737:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025935 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x644:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.853748:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02014c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x63b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0900 expref: 705 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.853757:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.853761:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35958 00010000:00010000:3.0:1783520935.853768:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc02018b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x644:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f097e expref: 705 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.853776:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943629897c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+705:2388:x1870153002402560:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.853778:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35959 00010000:00010000:3.0:1783520935.853789:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc02018b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x644:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f097e expref: 705 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.853791:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520935.853795:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d321c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+705:2388:x1870153002402688:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.853798:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d323800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+705:2388:x1870153002402304:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 194us (316us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.853803:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001f14d2b9/0xd5401cbabc02017d lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x642:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0962 expref: 705 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.853811:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.853819:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35957 00010000:00010000:2.0:1783520935.853823:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02014c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x63b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0900 expref: 705 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.853825:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e472300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993127040:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.853832:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001f14d2b9/0xd5401cbabc02017d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x642:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0962 expref: 705 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.853846:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02014c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x63b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0900 expref: 704 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.853851:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x642:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.853853:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025904 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.853860:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x642:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.853863:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435129679c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.853867:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02014c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x63b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0900 expref: 704 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.853870:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025927 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x642:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.853872:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025904 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.853895:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025927 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x642:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.853900:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025904 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.853908:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x63b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.853917:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x63b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.853923:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025904 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x63c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.853925:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0258f6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.853925:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.853947:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0258f6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x63b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.853950:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943629897c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+703:2388:x1870153002402560:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 178us (336us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.853955:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.853966:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35958 00010000:00010000:2.0:1783520935.853979:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.853996:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0258f6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.854005:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0258f6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.854014:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0258f6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.854019:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000016d99d15/0xd5401cbabc025919 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x640:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.854024:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0258f6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x63b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.854030:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000016d99d15/0xd5401cbabc025919 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x640:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.854043:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000016d99d15/0xd5401cbabc025919 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x640:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.854047:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000016d99d15/0xd5401cbabc025919 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x640:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.854049:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:0.0:1783520935.854056:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515776680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+298:3626:x1870152993124736:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5708us (5885us total) trans 4294975863 rc 0/0 00010000:00010000:1.0:1783520935.854063:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to regular blp list, 3 blwis in pool 00000100:00100000:0.0:1783520935.854063:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44361 00000004:00080000:3.0:1783520935.854072:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515775500/2, rc 0, transno 4294975863 00010000:00010000:1.0:1783520935.854075:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1783520935.854081:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025904 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x63c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.854087:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0258fd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.854089:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515775500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993124736:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.854091:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.854098:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025904 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x63c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.854106:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0258fd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.854110:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025904 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x63c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.854120:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0258fd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.854127:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025904 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x63c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.854134:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0258fd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x63d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.854139:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515777100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+298:3625:x1870152993124608:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6121us (6221us total) trans 4294975864 rc 0/0 00010000:00010000:1.0:1783520935.854141:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025904 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x63c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.854157:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44360 00000004:00080000:3.0:1783520935.854158:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515776d80/2, rc 0, transno 4294975864 00000100:00100000:3.0:1783520935.854172:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515776d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993124608:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.854175:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:1.0:1783520935.854181:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1783520935.854185:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0258f6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x63b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.854187:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520935.854189:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e472d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993127168:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.854195:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000016d99d15/0xd5401cbabc025919 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x640:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.854207:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.854211:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9435122ddc00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.854213:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0258f6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x63b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.854216:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000016d99d15/0xd5401cbabc025919 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x640:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.854228:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.854234:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0258f6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x63b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.854238:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000016d99d15/0xd5401cbabc025919 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x640:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.854253:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993127168 00010000:00010000:1.0:1783520935.854255:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0258f6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x63b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.854258:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000016d99d15/0xd5401cbabc025919 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x640:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.854260:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993127168 00000100:00100000:0.0:1783520935.854264:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e471180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993127296:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.854279:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0258f6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x63b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.854280:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000016d99d15/0xd5401cbabc025919 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x640:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.854294:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44370 00000100:00100000:0.0:1783520935.854303:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.854311:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350805ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+298:3628:x1870152993127168:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.854320:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.854321:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.854325:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00002000:00080000:3.0:1783520935.854326:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1605 count 1 00010000:00010000:2.0:1783520935.854326:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.854332:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.854332:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.854341:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02593c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x645:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.854344:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.854345:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.854353:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0258fd lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x63d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.854353:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993127296 00010000:00010000:3.0:1783520935.854385:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000590e7de8/0xd5401cbabc020192 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x645:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f098c expref: 702 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.854386:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993127296 00010000:00010000:2.0:1783520935.854400:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9435122dc400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.854406:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0258fd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x63d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.854410:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.854411:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x645:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.854421:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x645:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.854425:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44371 00010000:00010000:3.0:1783520935.854431:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02593c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x645:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.854434:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0258fd lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x63d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.854440:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350805b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+299:3628:x1870152993127296:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.854460:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0258fd lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x63d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.854460:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1606 count 1 00010000:00010000:2.0:1783520935.854471:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006266ad48/0xd5401cbabc0258fd lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x63d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.854473:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000590e7de8/0xd5401cbabc020192 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x645:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f098c expref: 702 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.854478:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006a6344f7/0xd5401cbabc025943 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x646:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.854510:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.854514:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.854516:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000590e7de8/0xd5401cbabc020192 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x645:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f098c expref: 702 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.854517:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000edaf1367/0xd5401cbabc022b30 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x646:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3220 expref: 702 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.854518:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.854535:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515774380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+300:3628:x1870152993124992:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6141us (6229us total) trans 4294975865 rc 0/0 00010000:00010000:1.0:1783520935.854543:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x646:0x0].0x0 since=36s v=(0 0) 00000100:00100000:2.0:1783520935.854546:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d321c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+702:2388:x1870153002402688:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 755us (881us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.854547:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e470380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993127424:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.854551:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x646:0x0].0x0 since=0s v=(1 0) 00000100:00100000:0.0:1783520935.854553:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44362 00010000:00010000:1.0:1783520935.854564:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006a6344f7/0xd5401cbabc025943 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x646:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.854565:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35959 00010000:00010000:1.0:1783520935.854594:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000edaf1367/0xd5401cbabc022b30 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x646:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3220 expref: 701 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.854595:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515774a80/2, rc 0, transno 4294975865 00000100:00100000:3.0:1783520935.854603:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967e40 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.854612:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000edaf1367/0xd5401cbabc022b30 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x646:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3220 expref: 701 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.854612:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515774a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993124992:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.854621:0:178496:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001f14d2b9/0xd5401cbabc02017d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x642:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0962 expref: 701 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.854637:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d323800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993127552:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.854643:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b9c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993126528:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.854665:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025927 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x642:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.854673:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351db63180 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.854677:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025927 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x642:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.854690:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025927 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x642:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.854700:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025927 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x642:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.854708:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025927 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x642:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.854728:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520935.854730:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350805b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993127680:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.854733:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.854741:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025927 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x642:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.854750:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.854753:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025927 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x642:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.854769:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025927 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x642:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.854774:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.854789:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025927 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x642:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.854798:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025927 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x642:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.854801:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993127680 00000100:00100000:2.0:1783520935.854807:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993127680 00010000:00010000:1.0:1783520935.854824:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.854828:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.854831:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.854840:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44372 00000100:00100000:2.0:1783520935.854850:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e471500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+302:3625:x1870152993127680:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.854864:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1607 count 1 00010000:00010000:2.0:1783520935.854877:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02594a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x647:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.854890:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc0201a0 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x647:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f09a8 expref: 700 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.854896:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x647:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.854900:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x647:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.854904:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e471c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+302:3626:x1870152993125760:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5592us (5709us total) trans 4294975866 rc 0/0 00010000:00010000:2.0:1783520935.854906:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02594a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x647:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.854914:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e470700/2, rc 0, transno 4294975866 00000100:00100000:1.0:1783520935.854920:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44365 00000100:00100000:3.0:1783520935.854930:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e470700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993125760:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.854942:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025912 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.854945:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc0201a0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x647:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09a8 expref: 700 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.854966:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025912 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.854970:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc0201a0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x647:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09a8 expref: 700 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.854983:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b9f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993126912:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.854988:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025912 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x63f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.854995:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b9c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993127808:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.855004:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc02592e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x643:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.855006:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025912 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x63f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.855030:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e470a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993127936:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.855036:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676c00 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.855037:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.855051:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.855058:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025912 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x63f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.855067:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.855073:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.855077:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025912 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x63f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.855087:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025912 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x63f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.855102:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025912 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x63f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.855109:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025912 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x63f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.855134:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.855137:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.855140:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.855140:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993127936 00000100:00100000:1.0:1783520935.855146:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993127936 00000100:00100000:0.0:1783520935.855146:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e472680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+302:3626:x1870152993125632:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6065us (6211us total) trans 4294975868 rc 0/0 00000100:00100000:0.0:1783520935.855157:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44364 00000100:00100000:0.0:1783520935.855181:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44373 00000100:00100000:2.0:1783520935.855194:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+302:3628:x1870152993126400:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5156us (5240us total) trans 4294975867 rc 0/0 00000100:00100000:2.0:1783520935.855202:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44367 00000004:00080000:0.0:1783520935.855210:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c5baa00/2, rc 0, transno 4294975867 00000100:00100000:1.0:1783520935.855212:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e473480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+301:3626:x1870152993127936:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.855225:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993126400:0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.855228:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1608 count 1 00010000:00010000:1.0:1783520935.855244:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025951 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x648:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.855246:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e471880/2, rc 0, transno 4294975868 00000100:00100000:3.0:1783520935.855256:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e471880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993125632:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.855263:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b95fce09/0xd5401cbabc0201a7 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x648:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f09b6 expref: 700 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.855271:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x648:0x0].0x0 since=37s v=(0 0) 00000100:00100000:0.0:1783520935.855277:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.855278:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x648:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.855285:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025951 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x648:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.855291:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993128064:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.855305:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993128192:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.855314:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b95fce09/0xd5401cbabc0201a7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x648:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09b6 expref: 700 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.855325:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b95fce09/0xd5401cbabc0201a7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x648:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09b6 expref: 700 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.855335:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.855345:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d320a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993128320:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.855387:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002402816 00000100:00100000:0.0:1783520935.855392:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002402816 00000100:00100000:0.0:1783520935.855408:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.855438:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993128064 00000100:00100000:2.0:1783520935.855443:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993128064 00000100:00100000:1.0:1783520935.855443:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57d80 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.855474:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993128192 00000100:00100000:2.0:1783520935.855478:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993128192 00010000:00010000:0.0:1783520935.855483:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc020184 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x643:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0970 expref: 701 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.855518:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44374 00000100:00100000:2.0:1783520935.855527:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5b8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+303:3627:x1870152993128064:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.855533:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35960 00002000:00080000:2.0:1783520935.855541:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1609 count 1 00000100:00100000:0.0:1783520935.855547:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508058a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+701:2387:x1870153002402816:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.855558:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025958 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x649:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.855560:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.855571:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc020184 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x643:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0970 expref: 701 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.855576:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0201ae lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x649:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f09c4 expref: 701 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.855591:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x649:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.855594:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc020184 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x643:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0970 expref: 701 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.855596:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x649:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.855604:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025958 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x649:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.855621:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc020184 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x643:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0970 expref: 701 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.855636:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x643:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.855637:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0201ae lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x649:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09c4 expref: 700 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.855643:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x643:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.855653:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc02592e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x643:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.855655:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0201ae lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x649:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09c4 expref: 700 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.855675:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc02592e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x643:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.855679:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b8380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993128448:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.855708:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.855725:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676180 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.855728:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44375 00000100:00100000:0.0:1783520935.855735:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508058a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+700:2387:x1870153002402816:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 191us (460us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.855742:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508059f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+303:3628:x1870152993128192:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.855745:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35960 00002000:00080000:1.0:1783520935.855761:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1610 count 1 00010000:00010000:0.0:1783520935.855775:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc02592e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x643:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.855777:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc02595f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x64a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.855792:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc02592e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x643:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.855801:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c0fb81b2/0xd5401cbabc0201b5 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x64a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f09d2 expref: 699 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.855804:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc02592e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x643:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.855823:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc02592e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x643:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.855834:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x64a:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.855839:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x64a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.855845:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc02595f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x64a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.855845:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.855875:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c0fb81b2/0xd5401cbabc0201b5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x64a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09d2 expref: 699 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.855885:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e472300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993127040:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.855889:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c0fb81b2/0xd5401cbabc0201b5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x64a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09d2 expref: 699 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.855904:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d323480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993128576:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.855909:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025935 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x644:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.855937:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.855942:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc02592e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x643:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.855948:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e470000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+304:3625:x1870152993126656:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2528us (2731us total) trans 4294975869 rc 0/0 00010000:00010000:2.0:1783520935.855951:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184800 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520935.855951:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511a4c6c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.855956:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc02592e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x643:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.855962:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44368 00010000:00010000:2.0:1783520935.855965:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc02592e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x643:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.855975:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc02592e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x643:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.855981:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005cc63f31/0xd5401cbabc02592e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x643:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.855997:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943508058380/2, rc 0, transno 4294975869 00010000:00010000:2.0:1783520935.855999:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.856004:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.856008:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.856009:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508058380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993126656:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.856072:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1783520935.856083:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508059500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993128704:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.856105:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.856145:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993128704 00000100:00100000:0.0:1783520935.856150:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993128704 00000100:00100000:0.0:1783520935.856182:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44376 00000100:00100000:0.0:1783520935.856189:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5b9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+304:3626:x1870152993128704:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.856201:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1611 count 1 00010000:00010000:0.0:1783520935.856213:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025966 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x64b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.856224:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc0201bc lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x64b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f09e0 expref: 699 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.856231:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x64b:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.856234:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x64b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.856238:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025966 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x64b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.856254:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc0201bc lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x64b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09e0 expref: 699 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.856262:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc0201bc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x64b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09e0 expref: 699 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.856271:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350805a680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993128832:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.856296:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.856629:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e470380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993127424:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.856637:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.856650:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02593c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x645:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.856724:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002402944 00000100:00100000:0.0:1783520935.856729:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002402944 00010000:00010000:0.0:1783520935.856746:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc02018b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x644:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f097e expref: 700 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.856771:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35961 00000100:00100000:0.0:1783520935.856779:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+700:2388:x1870153002402944:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.856787:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.856791:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc02018b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x644:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f097e expref: 700 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.856808:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc02018b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x644:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f097e expref: 700 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.856823:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc02018b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x644:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f097e expref: 700 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.856835:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x644:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.856838:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x644:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.856842:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025935 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x644:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.856850:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025935 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x644:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.856871:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.856894:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025935 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x644:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.856894:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+699:2388:x1870153002402944:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 116us (262us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.856909:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35961 00010000:00010000:1.0:1783520935.856912:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025935 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x644:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.856922:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025935 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x644:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.856931:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025935 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x644:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.856946:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.857022:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520935.857029:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b9c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993127808:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.857032:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025935 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x644:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.857042:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.857046:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025935 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x644:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.857047:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02594a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x647:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.857068:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025935 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x644:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.857069:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e473b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+305:3626:x1870152993126784:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3441us (3638us total) trans 4294975870 rc 0/0 00000100:00100000:1.0:1783520935.857086:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44369 00000100:00100000:0.0:1783520935.857092:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d323800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993127552:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.857093:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025935 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x644:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.857104:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025935 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x644:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.857113:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006a6344f7/0xd5401cbabc025943 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x646:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.857129:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520935.857131:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350805bb80/2, rc 0, transno 4294975870 00010000:00010000:2.0:1783520935.857133:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.857137:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.857151:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350805bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993126784:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.857185:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d320a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993128320:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.857202:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025951 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x648:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.857225:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.857305:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.857322:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002403072 00000100:00100000:0.0:1783520935.857326:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002403072 00010000:00010000:0.0:1783520935.857341:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000590e7de8/0xd5401cbabc020192 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x645:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f098c expref: 699 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.857343:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1783520935.857383:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508058e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993128960:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.857386:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002403328 00000100:00100000:0.0:1783520935.857390:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002403328 00010000:00010000:0.0:1783520935.857405:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b95fce09/0xd5401cbabc0201a7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x648:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09b6 expref: 700 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.857420:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.857442:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35962 00000100:00100000:0.0:1783520935.857452:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508058000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+700:2388:x1870153002403072:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.857460:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.857465:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000590e7de8/0xd5401cbabc020192 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x645:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f098c expref: 700 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.857474:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993128960 00010000:00010000:0.0:1783520935.857477:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000590e7de8/0xd5401cbabc020192 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x645:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f098c expref: 700 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.857479:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993128960 00010000:00010000:0.0:1783520935.857506:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000590e7de8/0xd5401cbabc020192 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x645:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f098c expref: 700 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.857524:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x645:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.857527:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x645:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.857531:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02593c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x645:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.857539:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02593c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x645:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.857556:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.857565:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02593c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x645:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.857572:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02593c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x645:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.857580:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02593c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x645:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.857586:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02593c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x645:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.857595:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44377 00010000:00010000:0.0:1783520935.857605:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.857608:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+305:3625:x1870152993128960:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.857621:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.857626:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02593c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x645:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.857635:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.857639:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02593c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x645:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.857648:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02593c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x645:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.857661:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02593c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x645:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.857670:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc02593c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x645:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.857693:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35963 00010000:00010000:1.0:1783520935.857696:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.857698:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350805ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+306:3628:x1870152993127168:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3390us (3471us total) trans 4294975871 rc 0/0 00010000:00010000:1.0:1783520935.857700:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.857705:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.857708:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350805aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+699:2388:x1870153002403328:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.857711:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44370 00002000:00080000:1.0:1783520935.857722:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1612 count 1 00010000:00010000:2.0:1783520935.857723:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.857732:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b95fce09/0xd5401cbabc0201a7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x648:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09b6 expref: 699 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.857740:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc02596d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x64c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.857746:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b95fce09/0xd5401cbabc0201a7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x648:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09b6 expref: 699 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.857751:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e472d80/2, rc 0, transno 4294975871 00010000:00010000:1.0:1783520935.857761:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000df223ebb/0xd5401cbabc0201c3 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x64c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f09ee expref: 699 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.857763:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e472d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993127168:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.857771:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b95fce09/0xd5401cbabc0201a7 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x648:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09b6 expref: 699 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.857776:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x64c:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.857784:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x64c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.857790:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x648:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.857790:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc02596d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x64c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.857799:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x648:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.857807:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025951 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x648:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.857812:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:2.0:1783520935.857817:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025951 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x648:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.857824:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000df223ebb/0xd5401cbabc0201c3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x64c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09ee expref: 698 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.857824:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508059c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993129088:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.857852:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000df223ebb/0xd5401cbabc0201c3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x64c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09ee expref: 698 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.857866:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.857867:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9ce00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993129216:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.857882:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.857917:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350805aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+698:2388:x1870153002403328:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 213us (609us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.857923:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508058000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+698:2388:x1870153002403072:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 471us (696us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.857928:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511a4c600 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.857930:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35963 00000100:00100000:0.0:1783520935.857939:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35962 00000100:00100000:1.0:1783520935.857952:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993129088 00000100:00100000:1.0:1783520935.857958:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993129088 00000100:00100000:1.0:1783520935.857984:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44378 00000100:00100000:1.0:1783520935.857994:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635603c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+306:3628:x1870152993129088:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.858008:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1613 count 1 00010000:00010000:1.0:1783520935.858021:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025974 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x64d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.858028:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025951 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x648:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.858035:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004a1917e6/0xd5401cbabc0201d1 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x64d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f09fc expref: 696 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.858040:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025951 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x648:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.858047:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x64d:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.858053:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x64d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.858054:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025951 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x648:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.858060:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025974 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x64d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.858070:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025951 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x648:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.858087:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.858089:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004a1917e6/0xd5401cbabc0201d1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x64d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09fc expref: 696 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.858097:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.858102:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004a1917e6/0xd5401cbabc0201d1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x64d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09fc expref: 696 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.858104:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025951 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x648:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.858126:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9ea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993129344:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.858129:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9435122dcc00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.858134:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025951 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x648:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.858147:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025951 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x648:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.858159:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025951 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x648:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.858166:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005ab0ba7b/0xd5401cbabc025951 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x648:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.858188:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.858191:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.858194:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.858251:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511a4cb40 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.858288:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e473480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+307:3626:x1870152993127936:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3097us (3222us total) trans 4294975872 rc 0/0 00000004:00080000:1.0:1783520935.858293:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e470a80/2, rc 0, transno 4294975872 00000100:00100000:2.0:1783520935.858298:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44373 00000100:00100000:1.0:1783520935.858310:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e470a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993127936:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.858324:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.858426:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.858448:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.858452:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1783520935.858467:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993129472:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.858470:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.858523:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.858523:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.858549:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002403200 00000100:00100000:0.0:1783520935.858553:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002403200 00000100:00100000:1.0:1783520935.858571:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002403584 00010000:00010000:0.0:1783520935.858572:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000edaf1367/0xd5401cbabc022b30 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x646:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3220 expref: 697 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.858578:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002403584 00000100:00100000:0.0:1783520935.858598:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002403712 00010000:00010000:1.0:1783520935.858603:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0201ae lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x649:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09c4 expref: 698 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.858604:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002403712 00000100:00100000:2.0:1783520935.858617:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002403456 00010000:00010000:0.0:1783520935.858622:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c0fb81b2/0xd5401cbabc0201b5 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x64a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09d2 expref: 699 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.858623:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002403456 00000100:00100000:1.0:1783520935.858637:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002403840 00000100:00100000:1.0:1783520935.858642:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002403840 00010000:00010000:2.0:1783520935.858647:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc0201a0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x647:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09a8 expref: 700 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.858651:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35964 00010000:00010000:1.0:1783520935.858662:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc0201bc lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x64b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09e0 expref: 701 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.858665:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508059880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+701:2388:x1870153002403200:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.858678:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.858685:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000edaf1367/0xd5401cbabc022b30 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x646:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3220 expref: 701 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.858691:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35965 00000100:00100000:1.0:1783520935.858701:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35966 00010000:00010000:0.0:1783520935.858704:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000edaf1367/0xd5401cbabc022b30 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x646:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3220 expref: 701 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.858706:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638584c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+701:2389:x1870153002403584:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.858718:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350805b800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+701:2390:x1870153002403712:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.858720:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.858728:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000edaf1367/0xd5401cbabc022b30 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x646:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3220 expref: 701 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.858729:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0201ae lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x649:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09c4 expref: 701 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.858732:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.858742:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c0fb81b2/0xd5401cbabc0201b5 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x64a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09d2 expref: 701 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.858752:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x646:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.858757:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0201ae lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x649:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09c4 expref: 700 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.858759:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x646:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.858765:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c0fb81b2/0xd5401cbabc0201b5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x64a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09d2 expref: 700 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.858769:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006a6344f7/0xd5401cbabc025943 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x646:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.858784:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x649:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.858791:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x649:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.858791:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006a6344f7/0xd5401cbabc025943 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x646:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.858796:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x64a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.858801:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025958 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x649:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.858805:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x64a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.858815:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc02595f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.858821:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.858828:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc02595f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x64a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.858847:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.858847:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508059880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+700:2388:x1870153002403200:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 186us (525us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.858856:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35964 00010000:00010000:2.0:1783520935.858865:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025958 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x649:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.858874:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35967 00000100:00100000:1.0:1783520935.858880:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350805b800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+699:2390:x1870153002403712:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 165us (410us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.858888:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.858888:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350805a300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+699:2390:x1870153002403456:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.858894:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35966 00010000:00010000:0.0:1783520935.858901:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.858908:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc0201a0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x647:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09a8 expref: 698 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.858916:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638584c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+698:2389:x1870153002403584:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 213us (486us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.858922:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35968 00000100:00100000:2.0:1783520935.858935:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35965 00000100:00100000:1.0:1783520935.858939:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508058700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+698:2387:x1870153002403840:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.858939:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc0201a0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x647:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09a8 expref: 698 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.858955:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.858960:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc0201a0 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x647:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09a8 expref: 697 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.858963:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc0201bc lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x64b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09e0 expref: 697 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.858980:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x647:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.858985:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc0201bc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x64b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09e0 expref: 696 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.858985:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x647:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.858993:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02594a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x647:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.858996:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993129472 00000100:00100000:2.0:1783520935.859001:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993129472 00010000:00010000:1.0:1783520935.859003:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x64b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.859005:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02594a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x647:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.859009:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x64b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.859019:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025966 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.859028:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025966 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x64b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.859033:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.859038:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44379 00010000:00010000:1.0:1783520935.859046:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.859051:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5b9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+307:3627:x1870152993129472:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.859051:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02594a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x647:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.859064:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02594a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x647:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.859070:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1614 count 1 00000100:00100000:1.0:1783520935.859072:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508058700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+696:2387:x1870153002403840:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 138us (551us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.859077:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02594a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x647:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.859087:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02597b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x64e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.859095:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35968 00010000:00010000:0.0:1783520935.859095:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02594a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x647:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.859109:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000055a5ddc5/0xd5401cbabc022a1f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x64e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f310f expref: 696 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.859114:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.859121:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x64e:0x0].0x0 since=36s v=(0 0) 00010000:00010000:2.0:1783520935.859125:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x64e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.859129:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02597b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x64e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.859141:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c0fb81b2/0xd5401cbabc0201b5 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x64a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09d2 expref: 695 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.859153:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000055a5ddc5/0xd5401cbabc022a1f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x64e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f310f expref: 694 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.859156:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d323480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993128576:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.859172:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000055a5ddc5/0xd5401cbabc022a1f lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x64e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f310f expref: 694 pid: 9151 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.859179:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc02595f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.859184:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516682300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993129600:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.859206:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e471500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+308:3625:x1870152993127680:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4356us (4433us total) trans 4294975873 rc 0/0 00000100:00100000:0.0:1783520935.859213:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44372 00000004:00080000:1.0:1783520935.859216:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350805b480/2, rc 0, transno 4294975873 00000100:00100000:1.0:1783520935.859230:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350805b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993127680:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.859243:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676540 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.859259:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.859264:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02594a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x647:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.859268:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350805a300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+694:2390:x1870153002403456:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 372us (806us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.859281:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185a00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520935.859283:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1783520935.859283:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35967 00010000:00010000:2.0:1783520935.859288:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02594a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x647:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.859296:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993129728:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.859299:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02594a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x647:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.859316:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006a6344f7/0xd5401cbabc025943 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x646:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.859317:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02594a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x647:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.859337:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.859340:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02594a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x647:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.859340:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006a6344f7/0xd5401cbabc025943 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x646:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.859389:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006a6344f7/0xd5401cbabc025943 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x646:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.859395:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.859398:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc02595f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.859399:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.859400:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006a6344f7/0xd5401cbabc025943 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x646:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.859404:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.859417:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993129728 00010000:00010000:1.0:1783520935.859420:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc02595f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.859423:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993129728 00010000:00010000:1.0:1783520935.859431:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc02595f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.859439:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc02595f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x64a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.859446:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.859456:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.859462:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.859464:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006a6344f7/0xd5401cbabc025943 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x646:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.859473:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.859475:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9435122dce00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.859479:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc02595f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x64a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.859481:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006a6344f7/0xd5401cbabc025943 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x646:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.859510:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9435122dd400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.859513:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006a6344f7/0xd5401cbabc025943 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x646:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.859516:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc02595f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x64a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.859541:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc02595f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x64a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.859544:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350805b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+309:3628:x1870152993127296:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5106us (5244us total) trans 4294975874 rc 0/0 00010000:00010000:2.0:1783520935.859546:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006a6344f7/0xd5401cbabc025943 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x646:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.859561:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44371 00010000:00010000:2.0:1783520935.859568:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006a6344f7/0xd5401cbabc025943 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x646:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.859570:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc02595f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x64a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.859581:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e7af8559/0xd5401cbabc02595f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x64a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.859581:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44380 00000100:00100000:0.0:1783520935.859593:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94361103b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+308:3626:x1870152993129728:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.859606:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1615 count 1 00010000:00010000:2.0:1783520935.859608:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.859612:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.859616:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.859620:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025982 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x64f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.859632:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f708e4ed/0xd5401cbabc0201df lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x64f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a18 expref: 693 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.859633:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e471180/2, rc 0, transno 4294975874 00010000:00010000:0.0:1783520935.859643:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x64f:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.859646:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x64f:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.859647:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e471180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993127296:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.859653:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025982 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x64f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.859656:0:20415:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0201ae lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x649:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09c4 expref: 693 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.859674:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b8380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993128448:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.859682:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f708e4ed/0xd5401cbabc0201df lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x64f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a18 expref: 692 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.859692:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025958 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.859694:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f708e4ed/0xd5401cbabc0201df lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x64f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a18 expref: 692 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.859696:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1783520935.859708:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e007800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993129984:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.859716:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025958 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.859722:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508058a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993129856:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.859731:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025958 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.859740:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.859743:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025958 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.859757:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025958 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.859767:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470b40 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.859774:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.859786:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.859787:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.859791:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025958 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.859791:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.859796:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.859800:0:9151:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc0201bc lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x64b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09e0 expref: 692 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.859803:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520935.859808:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993129984 00010000:00010000:2.0:1783520935.859810:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025958 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.859815:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993129984 00000100:00100000:0.0:1783520935.859819:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350805a680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993128832:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.859836:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025966 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.859845:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025966 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.859853:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44381 00010000:00010000:0.0:1783520935.859853:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025966 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.859864:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025966 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.859865:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e007b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+309:3625:x1870152993129984:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.859881:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025966 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x64b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.859882:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025958 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.859885:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1616 count 1 00010000:00010000:1.0:1783520935.859901:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025989 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x650:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.859907:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025958 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.859907:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.859918:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025958 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.859919:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0201d8 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x650:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a26 expref: 691 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.859937:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x650:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.859941:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x650:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.859944:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025989 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x650:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.859947:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.859954:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025966 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x64b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.859969:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.859972:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0201d8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x650:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a26 expref: 691 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.859974:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025966 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x64b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.859994:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025966 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x64b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.859996:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0201d8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x650:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a26 expref: 691 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.859996:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5b9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+310:3626:x1870152993128704:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3805us (3886us total) trans 4294975875 rc 0/0 00000100:00100000:0.0:1783520935.860013:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44376 00010000:00010000:2.0:1783520935.860022:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025966 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x64b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.860026:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e004e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993130112:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.860037:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025966 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x64b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.860055:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.860058:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.860061:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.860066:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435122b6c00 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.860069:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520935.860096:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943508059500/2, rc 0, transno 4294975875 00000100:00100000:1.0:1783520935.860108:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508059500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993128704:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.860149:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.860170:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1783520935.860179:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5b8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+310:3627:x1870152993128064:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4652us (4846us total) trans 4294975876 rc 0/0 00000100:00100000:1.0:1783520935.860182:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e005500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993130240:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.860193:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44374 00000100:00100000:0.0:1783520935.860201:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.860225:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:2.0:1783520935.860233:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c5b9f80/2, rc 0, transno 4294975876 00000100:00100000:2.0:1783520935.860249:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993128064:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.860278:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9ea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993129344:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.860285:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.860304:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025974 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x64d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.860307:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1783520935.860321:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516681880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993130368:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.860342:0:35278:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002404224 00000100:00100000:1.0:1783520935.860346:0:35278:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002404224 00000100:00100000:2.0:1783520935.860379:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.860402:0:35278:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98408 00000100:00100000:1.0:1783520935.860413:0:35278:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508058000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_009:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+26:251804:x1870153002404224:12345-192.168.204.33@tcp:101:dd.0 00000100:00100000:2.0:1783520935.860421:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002403968 00000100:00100000:2.0:1783520935.860427:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002403968 00010000:00010000:1.0:1783520935.860428:0:35278:0:(ldlm_lockd.c:1265:ldlm_handle_enqueue()) ### server-side enqueue handler START 00010000:00010000:1.0:1783520935.860443:0:35278:0:(ldlm_lockd.c:1343:ldlm_handle_enqueue()) ### server-side enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff9435122dd200/0xd5401cbabc025990 lrc: 2/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT gid 0 flags: 0x40000000000000 nid: local remote: 0xc275876b859f34e3 expref: -99 pid: 35278 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.860448:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000df223ebb/0xd5401cbabc0201c3 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x64c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09ee expref: 692 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.860482:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002404096 00000100:00100000:2.0:1783520935.860486:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002404096 00010000:00010000:2.0:1783520935.860513:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004a1917e6/0xd5401cbabc0201d1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x64d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09fc expref: 693 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.860537:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35969 00010000:00010000:1.0:1783520935.860545:0:35278:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff9435122dda00/0xd5401cbabc025997 lrc: 3/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x40000000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.860547:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350805a680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+693:2387:x1870153002403968:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.860560:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.860563:0:35278:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff9435122dda00/0xd5401cbabc025997 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT flags: 0x50210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.860568:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000df223ebb/0xd5401cbabc0201c3 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x64c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09ee expref: 693 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.860580:0:35278:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff9435122dda00/0xd5401cbabc025997 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.860585:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000df223ebb/0xd5401cbabc0201c3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x64c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09ee expref: 693 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.860601:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x64c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.860605:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x64c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.860611:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc02596d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.860619:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc02596d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x64c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00200000:1.0:1783520935.860625:0:35278:0:(mdt_handler.c:909:mdt_pack_attr2body()) [0x200000007:0x1:0x0]: returning size 69632 00000004:00200000:1.0:1783520935.860630:0:35278:0:(mdt_handler.c:1574:mdt_getattr_internal()) dirent count 0 stripe count 1 MDT count 2 00010000:00010000:2.0:1783520935.860646:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000004:00010000:1.0:1783520935.860651:0:35278:0:(mdt_handler.c:2507:mdt_getattr_name_lock()) ### Returning lock to client ns: mdt-lustre-MDT0000_UUID lock: ffff9435122dda00/0xd5401cbabc025997 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.860671:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350805a680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+693:2387:x1870153002403968:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 127us (523us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.860677:0:35278:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff9435122dd200/0xd5401cbabc025990 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x44000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34e3 expref: 28 pid: 35278 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.860684:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35969 00010000:00010000:1.0:1783520935.860702:0:35278:0:(ldlm_lockd.c:1513:ldlm_handle_enqueue()) ### server-side enqueue handler, sending reply (err=0, rc=0) ns: mdt-lustre-MDT0000_UUID lock: ffff9435122dda00/0xd5401cbabc025997 lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x40200000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34e3 expref: 27 pid: 35278 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.860707:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35970 00010000:00010000:1.0:1783520935.860716:0:35278:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x0 (ffff943504b72c00) 00000100:00100000:2.0:1783520935.860719:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350805aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+692:2390:x1870153002404096:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.860720:0:35278:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 13 00010000:00010000:1.0:1783520935.860725:0:35278:0:(ldlm_lockd.c:1592:ldlm_handle_enqueue()) ### server-side enqueue handler END (lock ffff9435122dda00, rc 0) 00010000:00010000:2.0:1783520935.860731:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.860738:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004a1917e6/0xd5401cbabc0201d1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x64d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09fc expref: 692 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.860750:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004a1917e6/0xd5401cbabc0201d1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x64d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09fc expref: 692 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.860754:0:35278:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508058000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_009:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+27:251804:x1870153002404224:12345-192.168.204.33@tcp:101:dd.0 Request processed in 342us (471us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.860768:0:35278:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98408 00010000:00010000:2.0:1783520935.860773:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004a1917e6/0xd5401cbabc0201d1 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x64d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09fc expref: 692 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.860786:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x64d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.860790:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x64d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.860795:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025974 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.860803:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025974 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x64d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.860840:0:35276:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000df223ebb/0xd5401cbabc0201c3 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x64c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f09ee expref: 691 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.860850:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9ce00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993129216:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.860863:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc02596d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.860869:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc02596d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.860874:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc02596d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.860883:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.860883:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc02596d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.860890:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc02596d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x64c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.860902:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520935.860910:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350805aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+690:2390:x1870153002404096:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 193us (709us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.860919:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35970 00010000:00010000:2.0:1783520935.860940:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.860945:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc02596d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x64c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.860952:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9435122dd600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.860956:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc02596d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x64c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.860964:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc02596d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x64c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.860976:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc02596d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x64c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.860983:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000046245395/0xd5401cbabc02596d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x64c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.860991:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+310:3625:x1870152993128960:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3387us (3569us total) trans 4294975877 rc 0/0 00000100:00100000:1.0:1783520935.860997:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44377 00010000:00010000:2.0:1783520935.861003:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.861007:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520935.861008:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943508058e00/2, rc 0, transno 4294975877 00010000:00010000:2.0:1783520935.861010:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.861016:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993130240 00000100:00100000:1.0:1783520935.861021:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993130240 00000100:00100000:0.0:1783520935.861028:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508058e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993128960:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.861047:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993130368 00000100:00100000:1.0:1783520935.861052:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993130368 00000100:00100000:0.0:1783520935.861064:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516682d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993130496:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.861076:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44382 00010000:00010000:1.0:1783520935.861084:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025974 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.861088:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e007480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+311:3628:x1870152993130240:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.861101:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.861103:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025974 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.861105:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1617 count 1 00010000:00010000:1.0:1783520935.861116:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025974 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.861120:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02599e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x651:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.861128:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025974 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.861135:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000006bc0c1a/0xd5401cbabc0201e6 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x651:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a34 expref: 689 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.861157:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x651:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.861162:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x651:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.861165:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02599e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x651:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.861167:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.861176:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.861182:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025974 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.861191:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9435122dc600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.861192:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000006bc0c1a/0xd5401cbabc0201e6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x651:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a34 expref: 689 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.861197:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025974 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.861212:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000006bc0c1a/0xd5401cbabc0201e6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x651:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a34 expref: 689 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.861213:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025974 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.861236:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b8380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993130624:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.861239:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025974 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.861249:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000078555dd3/0xd5401cbabc025974 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.861268:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.861271:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.861272:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635603c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+312:3628:x1870152993129088:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3277us (3406us total) trans 4294975878 rc 0/0 00010000:00010000:0.0:1783520935.861274:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.861284:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44378 00000100:00100000:2.0:1783520935.861289:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676180 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.861308:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993130496 00000100:00100000:2.0:1783520935.861312:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993130496 00000004:00080000:1.0:1783520935.861323:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943508059c00/2, rc 0, transno 4294975878 00000100:00100000:1.0:1783520935.861337:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508059c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993129088:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.861342:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44383 00000100:00100000:2.0:1783520935.861352:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508059880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+312:3625:x1870152993130496:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.861410:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00002000:00080000:2.0:1783520935.861418:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1619 count 1 00000100:00100000:1.0:1783520935.861419:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993130752:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.861435:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0259a5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x653:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.861448:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e34a9273/0xd5401cbabc0201f4 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x653:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a50 expref: 689 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.861456:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.861462:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x653:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.861465:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x653:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.861469:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0259a5 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x653:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.861489:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993130752 00010000:00010000:2.0:1783520935.861490:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e34a9273/0xd5401cbabc0201f4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x653:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a50 expref: 689 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.861505:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993130752 00010000:00010000:2.0:1783520935.861516:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e34a9273/0xd5401cbabc0201f4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x653:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a50 expref: 689 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.861527:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f1180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993130880:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.861533:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44384 00000100:00100000:1.0:1783520935.861541:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+313:3625:x1870152993130752:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.861553:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1620 count 1 00010000:00010000:1.0:1783520935.861565:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0259ac lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x654:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.861567:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676c00 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.861580:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c275a401/0xd5401cbabc0201fb lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x654:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a5e expref: 689 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.861589:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x654:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.861592:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x654:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.861597:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0259ac lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x654:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.861623:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c275a401/0xd5401cbabc0201fb lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x654:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a5e expref: 689 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.861632:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c275a401/0xd5401cbabc0201fb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x654:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a5e expref: 689 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.861642:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993131008:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.861672:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f000 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.861721:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44385 00000100:00100000:1.0:1783520935.861730:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516682a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+313:3627:x1870152993130368:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.861744:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1618 count 1 00010000:00010000:1.0:1783520935.861757:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc0259b3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x652:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.861769:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0201ed lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x652:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a42 expref: 689 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.861778:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x652:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.861781:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x652:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.861785:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc0259b3 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x652:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.861824:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e004e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993130112:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.861843:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025989 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x650:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.861858:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508058a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993129856:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.861879:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025982 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x64f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.861898:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516682300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993129600:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.861918:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02597b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x64e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.861958:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508059f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+314:3628:x1870152993128192:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6220us (6550us total) trans 4294975879 rc 0/0 00000100:00100000:1.0:1783520935.861965:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44375 00000004:00080000:0.0:1783520935.862069:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c5bb800/2, rc 0, transno 4294975879 00010000:00010000:1.0:1783520935.862083:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0201ed lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x652:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a42 expref: 689 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.862087:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993128192:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.862101:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0201ed lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x652:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a42 expref: 689 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.862112:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993131136:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.862148:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511a4c600 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.862154:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516680e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993131264:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.862188:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.862237:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993131264 00000100:00100000:2.0:1783520935.862242:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993131264 00000100:00100000:1.0:1783520935.862250:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.862274:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44386 00000100:00100000:0.0:1783520935.862279:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002404480 00000100:00100000:2.0:1783520935.862285:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516682300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+314:3627:x1870152993131264:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.862286:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002404480 00002000:00080000:2.0:1783520935.862302:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1621 count 1 00010000:00010000:0.0:1783520935.862313:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f708e4ed/0xd5401cbabc0201df lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x64f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a18 expref: 690 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.862318:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0259ba lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x655:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.862338:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc020202 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x655:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a6c expref: 690 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.862346:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x655:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.862351:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x655:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.862353:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35971 00010000:00010000:2.0:1783520935.862383:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0259ba lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x655:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.862387:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c2d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+690:2388:x1870153002404480:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.862403:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.862410:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f708e4ed/0xd5401cbabc0201df lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x64f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a18 expref: 690 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.862419:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc020202 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x655:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a6c expref: 690 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.862427:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f708e4ed/0xd5401cbabc0201df lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x64f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a18 expref: 690 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.862435:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc020202 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x655:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a6c expref: 690 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.862451:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517244e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993131392:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.862453:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f708e4ed/0xd5401cbabc0201df lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x64f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a18 expref: 690 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.862469:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x64f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.862473:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x64f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.862479:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025982 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.862488:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025982 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x64f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.862517:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676540 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.862519:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.862543:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c2d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+689:2388:x1870153002404480:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 160us (294us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.862553:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35971 00010000:00010000:0.0:1783520935.862582:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025982 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.862587:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.862596:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025982 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.862606:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025982 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.862615:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025982 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x64f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.862632:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.862655:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.862662:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025982 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x64f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.862677:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.862681:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025982 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x64f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.862690:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025982 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x64f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.862704:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025982 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x64f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.862712:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025982 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x64f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.862730:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94361103b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+315:3626:x1870152993129728:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3138us (3395us total) trans 4294975880 rc 0/0 00010000:00010000:2.0:1783520935.862736:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.862741:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.862741:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44380 00000004:00080000:1.0:1783520935.862744:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9d180/2, rc 0, transno 4294975880 00010000:00010000:2.0:1783520935.862747:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.862765:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993129728:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.862776:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002404352 00000100:00100000:0.0:1783520935.862781:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002404352 00010000:00010000:0.0:1783520935.862801:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0201d8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x650:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a26 expref: 689 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.862809:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514f9ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993131520:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.862833:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35972 00000100:00100000:1.0:1783520935.862840:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.862844:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943629897450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+689:2388:x1870153002404352:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.862856:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.862861:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0201d8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x650:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a26 expref: 689 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.862874:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0201d8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x650:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a26 expref: 689 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.862881:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993131520 00000100:00100000:1.0:1783520935.862886:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993131520 00010000:00010000:0.0:1783520935.862892:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0201d8 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x650:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a26 expref: 689 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.862907:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x650:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.862911:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x650:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.862916:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025989 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x650:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.862925:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44387 00010000:00010000:0.0:1783520935.862927:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025989 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x650:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.862936:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+315:3628:x1870152993131520:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.862943:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b8380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993130624:192.168.204.33@tcp:104: 00002000:00080000:1.0:1783520935.862952:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1622 count 1 00010000:00010000:0.0:1783520935.862953:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.862963:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02599e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x651:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.862970:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025989 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x650:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.862972:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc0259c1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x656:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.862990:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025989 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x650:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.862998:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f1180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993130880:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.862998:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002137d0eb/0xd5401cbabc02290e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x656:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ffe expref: 688 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.863006:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025989 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x650:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.863018:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0259a5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x653:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.863024:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x656:0x0].0x0 since=36s v=(0 0) 00010000:00010000:1.0:1783520935.863037:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x656:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.863048:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025989 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x650:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.863049:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc0259c1 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x656:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.863081:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.863090:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002137d0eb/0xd5401cbabc02290e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x656:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ffe expref: 688 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.863092:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.863100:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025989 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x650:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.863107:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002137d0eb/0xd5401cbabc02290e lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x656:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ffe expref: 688 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.863112:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9435122dc800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.863118:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025989 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x650:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.863126:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c0e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993131648:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.863130:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025989 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x650:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.863144:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025989 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x650:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.863152:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000ef9e284/0xd5401cbabc025989 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x650:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.863161:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511a4cb40 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.863179:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.863182:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.863188:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e007b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+316:3625:x1870152993129984:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3324us (3447us total) trans 4294975881 rc 0/0 00010000:00010000:2.0:1783520935.863191:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.863198:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44381 00000004:00080000:0.0:1783520935.863242:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e007800/2, rc 0, transno 4294975881 00000100:00100000:0.0:1783520935.863258:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e007800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993129984:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.863305:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943629897450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+688:2388:x1870153002404352:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 462us (718us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.863314:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1783520935.863315:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35972 00000100:00100000:1.0:1783520935.863327:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514f99c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993131776:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.863427:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.863466:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993131776 00000100:00100000:0.0:1783520935.863471:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993131776 00000100:00100000:0.0:1783520935.863518:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44388 00000100:00100000:0.0:1783520935.863529:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+316:3626:x1870152993131776:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.863542:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1623 count 1 00010000:00010000:0.0:1783520935.863556:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0259c8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x657:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.863569:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004e921a45/0xd5401cbabc020209 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x657:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a88 expref: 687 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.863577:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x657:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.863581:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x657:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.863586:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0259c8 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x657:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.863610:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004e921a45/0xd5401cbabc020209 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x657:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a88 expref: 687 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.863622:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004e921a45/0xd5401cbabc020209 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x657:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a88 expref: 687 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.863635:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350805b800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993131904:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.863687:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470900 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.864129:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.864146:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.864167:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002404736 00000100:00100000:0.0:1783520935.864173:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002404736 00000100:00100000:2.0:1783520935.864176:0:35279:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002405120 00000100:00100000:2.0:1783520935.864181:0:35279:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002405120 00000100:00100000:1.0:1783520935.864183:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.864197:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e34a9273/0xd5401cbabc0201f4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x653:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a50 expref: 688 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.864218:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.864231:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002404608 00000100:00100000:0.0:1783520935.864235:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002404608 00000100:00100000:1.0:1783520935.864246:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002404864 00000100:00100000:1.0:1783520935.864251:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002404864 00010000:00010000:0.0:1783520935.864255:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000055a5ddc5/0xd5401cbabc022a1f lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x64e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f310f expref: 689 pid: 9151 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.864274:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000006bc0c1a/0xd5401cbabc0201e6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x651:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a34 expref: 690 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.864293:0:35279:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98409 00000100:00100000:0.0:1783520935.864307:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35973 00000100:00100000:2.0:1783520935.864308:0:35279:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638494c50 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_010:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+27:251804:x1870153002405120:12345-192.168.204.33@tcp:101:dd.0 00000100:00100000:1.0:1783520935.864318:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35974 00010000:00010000:2.0:1783520935.864324:0:35279:0:(ldlm_lockd.c:1265:ldlm_handle_enqueue()) ### server-side enqueue handler START 00000100:00100000:0.0:1783520935.864324:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c0000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+690:2390:x1870153002404736:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.864329:0:35279:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 2 locks, starting at 1 00000100:00100000:1.0:1783520935.864335:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c3800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+690:2387:x1870153002404608:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.864339:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.864346:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.864347:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e34a9273/0xd5401cbabc0201f4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x653:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a50 expref: 690 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.864352:0:35279:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff9435122dda00/0xd5401cbabc025997 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x44a01000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34e3 expref: 27 pid: 35278 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.864383:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000055a5ddc5/0xd5401cbabc022a1f lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x64e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f310f expref: 690 pid: 9151 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.864399:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000055a5ddc5/0xd5401cbabc022a1f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x64e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f310f expref: 690 pid: 9151 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.864408:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e34a9273/0xd5401cbabc0201f4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x653:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a50 expref: 690 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.864435:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000055a5ddc5/0xd5401cbabc022a1f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x64e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f310f expref: 690 pid: 9151 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.864437:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e34a9273/0xd5401cbabc0201f4 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x653:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a50 expref: 690 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.864464:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x653:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.864466:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x64e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.864469:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x653:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.864476:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0259a5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x653:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.864484:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0259a5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x653:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.864490:0:35279:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x0 (ffff943504b72c00) 00010000:00010000:2.0:1783520935.864505:0:35279:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.864519:0:35279:0:(ldlm_lockd.c:1343:ldlm_handle_enqueue()) ### server-side enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943512184200/0xd5401cbabc0259cf lrc: 2/0,0 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT gid 0 flags: 0x40000000000000 nid: local remote: 0xc275876b859f34ea expref: -99 pid: 35279 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.864520:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.864553:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c0000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+688:2390:x1870153002404736:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 234us (407us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.864563:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35973 00010000:00010000:1.0:1783520935.864566:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x64e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.864572:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02597b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.864579:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02597b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x64e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.864585:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35975 00000100:00100000:0.0:1783520935.864598:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c2d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+687:2387:x1870153002404864:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.864608:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.864610:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.864615:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000006bc0c1a/0xd5401cbabc0201e6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x651:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a34 expref: 687 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.864634:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000006bc0c1a/0xd5401cbabc0201e6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x651:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a34 expref: 687 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.864636:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c3800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+687:2387:x1870153002404608:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 304us (451us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.864653:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35974 00010000:00010000:0.0:1783520935.864654:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000006bc0c1a/0xd5401cbabc0201e6 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x651:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a34 expref: 687 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.864668:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x651:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.864672:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x651:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.864676:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02599e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x651:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.864685:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02599e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x651:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.864706:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.864731:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c2d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+685:2387:x1870153002404864:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 136us (512us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.864739:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35975 00010000:00010000:2.0:1783520935.864746:0:35279:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff943512184c00/0xd5401cbabc0259d6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x40000000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.864760:0:35279:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943512184c00/0xd5401cbabc0259d6 lrc: 3/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.864788:0:35279:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943512184c00/0xd5401cbabc0259d6 lrc: 3/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.864804:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993131008:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.864807:0:35279:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff943512185800/0xd5401cbabc0259dd lrc: 3/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x40000000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.864818:0:35279:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943512185800/0xd5401cbabc0259dd lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.864826:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0259ac lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x654:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.864842:0:35279:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943512185800/0xd5401cbabc0259dd lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.864931:0:35279:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff943512185c00/0xd5401cbabc0259e4 lrc: 3/1,0 mode: --/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x0/0x0 rrc: 7 type: IBT flags: 0x40000000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.864943:0:35279:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943512185c00/0xd5401cbabc0259e4 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 7 type: IBT flags: 0x50210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.864955:0:35279:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943512185c00/0xd5401cbabc0259e4 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 7 type: IBT flags: 0x40210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00000004:00200000:2.0:1783520935.864967:0:35279:0:(mdt_handler.c:909:mdt_pack_attr2body()) [0x200000402:0x72a:0x0]: returning size 262144 00000100:00100000:1.0:1783520935.864986:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.865019:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993131136:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.865020:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.865042:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc0259b3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x652:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.865060:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.865078:0:35279:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff943512184c00/0xd5401cbabc0259d6 lrc: 4/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.865078:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002405248 00000100:00100000:0.0:1783520935.865084:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002405248 00010000:00010000:2.0:1783520935.865092:0:35279:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff943512184c00/0xd5401cbabc0259d6 lrc: 4/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.865099:0:35279:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff943512184c00/0xd5401cbabc0259d6 lrc: 3/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.865105:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c275a401/0xd5401cbabc0201fb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x654:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a5e expref: 685 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.865110:0:35279:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff943512184c00/0xd5401cbabc0259d6 lrc: 4/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.865111:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002405376 00000100:00100000:1.0:1783520935.865116:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002405376 00010000:00010000:2.0:1783520935.865122:0:35279:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184c00 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:2.0:1783520935.865126:0:35279:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943512184c00/0xd5401cbabc0259d6 lrc: 4/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.865128:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002405504 00000100:00100000:0.0:1783520935.865132:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002405504 00010000:00010000:1.0:1783520935.865134:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc020202 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x655:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a6c expref: 686 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.865136:0:35279:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943512184c00/0xd5401cbabc0259d6 lrc: 5/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40218400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.865147:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0201ed lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x652:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a42 expref: 687 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.865156:0:35279:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x0 (ffff943504b72c00) 00010000:00010000:2.0:1783520935.865160:0:35279:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:2.0:1783520935.865163:0:35279:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff943512184c00/0xd5401cbabc0259d6 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x44a19400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.865169:0:35279:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943512184c00/0xd5401cbabc0259d6 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x44a19400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.865176:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35976 00010000:00010000:2.0:1783520935.865179:0:35279:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff943512185800/0xd5401cbabc0259dd lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.865184:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35977 00000100:00100000:1.0:1783520935.865187:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+687:2387:x1870153002405248:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.865196:0:35279:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff943512185800/0xd5401cbabc0259dd lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.865198:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506228a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+687:2388:x1870153002405376:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.865201:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.865209:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.865210:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c275a401/0xd5401cbabc0201fb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x654:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a5e expref: 687 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.865220:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc020202 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x655:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a6c expref: 687 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.865223:0:35279:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff943512185800/0xd5401cbabc0259dd lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.865240:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c275a401/0xd5401cbabc0201fb lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x654:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a5e expref: 687 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.865245:0:35279:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:0.0:1783520935.865250:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc020202 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x655:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a6c expref: 687 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.865252:0:35279:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943512185800/0xd5401cbabc0259dd lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.865260:0:35279:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943512185800/0xd5401cbabc0259dd lrc: 4/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40218400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.865263:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c275a401/0xd5401cbabc0201fb lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x654:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a5e expref: 687 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.865287:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x654:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.865290:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x655:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.865292:0:35279:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x5ba6781 (ffff94351e3a6400) 00010000:00010000:1.0:1783520935.865292:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x654:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.865295:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x655:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.865299:0:35279:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:0.0:1783520935.865306:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0259ba lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x655:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.865307:0:35279:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff943512185800/0xd5401cbabc0259dd lrc: 2/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.865308:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0259ac lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x654:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.865326:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0259ba lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x655:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.865329:0:35279:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943512185800/0xd5401cbabc0259dd lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 35279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.865335:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0259ac lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x654:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.865351:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.865387:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.865401:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506228a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+686:2388:x1870153002405376:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 207us (383us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.865406:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35977 00000100:00100000:1.0:1783520935.865413:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+686:2387:x1870153002405248:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 228us (426us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.865423:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35976 00010000:00010000:2.0:1783520935.865424:0:35279:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943512184200/0xd5401cbabc0259cf lrc: 0/0,0 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x44000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34ea expref: 29 pid: 35279 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.865432:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35978 00000100:00100000:0.0:1783520935.865446:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634571c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+684:2389:x1870153002405504:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.865448:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0259ac lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x654:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.865451:0:35279:0:(ldlm_lockd.c:1513:ldlm_handle_enqueue()) ### server-side enqueue handler, sending reply (err=0, rc=0) ns: mdt-lustre-MDT0000_UUID lock: ffff943512185c00/0xd5401cbabc0259e4 lrc: 3/0,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 7 type: IBT gid 0 flags: 0x40200000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34ea expref: 28 pid: 35279 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.865461:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.865471:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0201ed lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x652:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a42 expref: 684 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.865472:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0259ac lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x654:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.865473:0:35279:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x72a:0x0].0x0 (ffff943515473e00) 00010000:00010000:2.0:1783520935.865479:0:35279:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 1b 00010000:00010000:2.0:1783520935.865484:0:35279:0:(ldlm_lockd.c:1592:ldlm_handle_enqueue()) ### server-side enqueue handler END (lock ffff943512185c00, rc 0) 00010000:00010000:0.0:1783520935.865493:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0201ed lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x652:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a42 expref: 684 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.865506:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0259ac lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x654:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.865525:0:35279:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638494c50 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_010:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+28:251804:x1870153002405120:12345-192.168.204.33@tcp:101:dd.0 Request processed in 1222us (1396us total) trans 8589957898 rc 0/0 00010000:00010000:1.0:1783520935.865529:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0259ac lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x654:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.865534:0:35279:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98409 00010000:00010000:0.0:1783520935.865535:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0201ed lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x652:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a42 expref: 684 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.865558:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x652:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.865560:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x652:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.865561:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.865564:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc0259b3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x652:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.865571:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc0259b3 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x652:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.865586:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.865608:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634571c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+683:2389:x1870153002405504:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 165us (547us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.865618:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35978 00010000:00010000:0.0:1783520935.865653:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc0259b3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x652:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.865661:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc0259b3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x652:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.865669:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc0259b3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x652:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.865677:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc0259b3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x652:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.865685:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+317:3625:x1870152993130752:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4142us (4229us total) trans 4294975882 rc 0/0 00000100:00100000:1.0:1783520935.865696:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44384 00000004:00080000:0.0:1783520935.865721:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9ce00/2, rc 0, transno 4294975882 00000100:00100000:0.0:1783520935.865737:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993130752:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.865747:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c0e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993131648:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.865769:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc0259c1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x656:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.865804:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508058700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993132032:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.865822:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:0.0:1783520935.865834:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00000100:00100000:1.0:1783520935.865839:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.865842:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0259ac lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x654:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.865853:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9435122dde00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.865857:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0259ac lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x654:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.865865:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0259ac lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x654:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.865879:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0259ac lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x654:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.865887:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b63fac49/0xd5401cbabc0259ac lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x654:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.865904:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.865915:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.865920:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc0259b3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x652:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.865927:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9435122dca00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.865931:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc0259b3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x652:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.865939:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc0259b3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x652:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.865951:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc0259b3 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x652:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.865958:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004013ce7d/0xd5401cbabc0259b3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x652:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.865980:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.865984:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.865988:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.865990:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.866003:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993132032 00000100:00100000:0.0:1783520935.866008:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993132032 00000100:00100000:0.0:1783520935.866046:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44389 00000100:00100000:0.0:1783520935.866056:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634571f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+317:3626:x1870152993132032:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.866073:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1624 count 1 00010000:00010000:0.0:1783520935.866090:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0259f2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x658:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.866103:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000019472967/0xd5401cbabc020210 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x658:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a96 expref: 682 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.866110:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x658:0x0].0x0 since=37s v=(0 0) 00000100:00100000:1.0:1783520935.866111:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002405632 00010000:00010000:0.0:1783520935.866116:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x658:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.866117:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002405632 00010000:00010000:0.0:1783520935.866124:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0259f2 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x658:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.866145:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002137d0eb/0xd5401cbabc02290e lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x656:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ffe expref: 683 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.866149:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000019472967/0xd5401cbabc020210 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x658:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a96 expref: 683 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.866165:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000019472967/0xd5401cbabc020210 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x658:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a96 expref: 683 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.866170:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002405760 00000100:00100000:1.0:1783520935.866176:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002405760 00000100:00100000:0.0:1783520935.866180:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350805a300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993132160:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.866194:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004e921a45/0xd5401cbabc020209 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x657:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a88 expref: 684 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.866225:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35979 00000100:00100000:1.0:1783520935.866236:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943629895850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+684:2387:x1870153002405632:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.866245:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.866245:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470b40 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.866255:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002137d0eb/0xd5401cbabc02290e lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x656:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ffe expref: 684 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.866268:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002137d0eb/0xd5401cbabc02290e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x656:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ffe expref: 684 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.866273:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35980 00000100:00100000:0.0:1783520935.866288:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634573b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+684:2387:x1870153002405760:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.866295:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002137d0eb/0xd5401cbabc02290e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x656:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ffe expref: 684 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.866301:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.866307:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x656:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.866309:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004e921a45/0xd5401cbabc020209 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x657:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a88 expref: 683 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.866312:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x656:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.866320:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc0259c1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x656:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.866327:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc0259c1 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x656:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.866329:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004e921a45/0xd5401cbabc020209 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x657:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a88 expref: 683 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.866349:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.866351:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x657:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.866355:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x657:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.866383:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0259c8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x657:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.866392:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0259c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x657:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.866397:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943629895850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+683:2387:x1870153002405632:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 163us (494us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.866411:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35979 00010000:00010000:0.0:1783520935.866418:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.866446:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634573b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+682:2387:x1870153002405760:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 161us (457us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.866455:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35980 00010000:00010000:1.0:1783520935.866478:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc0259c1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x656:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.866489:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc0259c1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x656:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.866510:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc0259c1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x656:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.866517:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc0259c1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x656:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.866533:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.866602:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516682a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+319:3627:x1870152993130368:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4874us (6246us total) trans 4294975883 rc 0/0 00000100:00100000:0.0:1783520935.866612:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44385 00000100:00100000:1.0:1783520935.866634:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+318:3628:x1870152993131520:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3699us (3792us total) trans 4294975884 rc 0/0 00000100:00100000:1.0:1783520935.866643:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44387 00000004:00080000:0.0:1783520935.866660:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516681880/2, rc 0, transno 4294975883 00000100:00100000:0.0:1783520935.866673:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516681880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993130368:0@lo:6:osp-syn-1-0.0 00000004:00080000:1.0:1783520935.866686:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514f9ad80/2, rc 0, transno 4294975884 00000100:00100000:1.0:1783520935.866695:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514f9ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993131520:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.866737:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1783520935.866751:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350805aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993132416:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.866785:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.866788:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004e921a45/0xd5401cbabc020209 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x657:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a88 expref: 681 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.866809:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350805b800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993131904:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.866820:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350805a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993132288:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.866830:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0259c8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x657:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.866850:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.866852:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993132416 00000100:00100000:0.0:1783520935.866858:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993132416 00000100:00100000:0.0:1783520935.866891:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993132288 00000100:00100000:0.0:1783520935.866894:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993132288 00000100:00100000:1.0:1783520935.866898:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44390 00000100:00100000:1.0:1783520935.866910:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634571c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+319:3628:x1870152993132416:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.866915:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44391 00000100:00100000:0.0:1783520935.866923:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506228a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+319:3628:x1870152993132288:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.866924:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1626 count 1 00002000:00080000:0.0:1783520935.866937:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1625 count 1 00010000:00010000:1.0:1783520935.866942:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0259f9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x65a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.866950:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025a00 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x659:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.866957:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007c066493/0xd5401cbabc02021e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x65a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ab2 expref: 680 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.866963:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000099edab59/0xd5401cbabc020217 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x659:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0aa4 expref: 680 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.866976:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x65a:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.866979:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x659:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.866980:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x65a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.866987:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0259f9 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x65a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.866987:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x659:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.866996:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025a00 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x659:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.867036:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000099edab59/0xd5401cbabc020217 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x659:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0aa4 expref: 680 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.867042:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007c066493/0xd5401cbabc02021e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x65a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ab2 expref: 680 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.867060:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000099edab59/0xd5401cbabc020217 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x659:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0aa4 expref: 680 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.867065:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007c066493/0xd5401cbabc02021e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x65a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ab2 expref: 680 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.867075:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350805b800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993132544:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.867083:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515777480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993132672:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.867116:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470900 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.867118:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511a4c600 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.867131:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0259c8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x657:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.867137:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0259c8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x657:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.867143:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0259c8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x657:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.867148:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0259c8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x657:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.867162:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:0.0:1783520935.867171:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520935.867176:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc0259c1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x656:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.867182:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.867186:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc0259c1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x656:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.867192:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc0259c1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x656:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.867201:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc0259c1 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x656:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.867206:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc0259c1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x656:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.867226:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.867229:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0259c8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x657:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.867234:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.867237:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0259c8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x657:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.867242:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0259c8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x657:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.867250:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0259c8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x657:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.867254:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0259c8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x657:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.867277:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.867279:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.867282:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.867391:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+320:3626:x1870152993131776:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3865us (4032us total) trans 4294975885 rc 0/0 00000100:00100000:0.0:1783520935.867400:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44388 00000004:00080000:1.0:1783520935.867420:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514f99c00/2, rc 0, transno 4294975885 00000100:00100000:1.0:1783520935.867432:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514f99c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993131776:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.867483:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1783520935.867506:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508058a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993132800:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.867537:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.867565:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993132800 00000100:00100000:0.0:1783520935.867571:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993132800 00000100:00100000:0.0:1783520935.867604:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44392 00000100:00100000:0.0:1783520935.867614:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515776680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+320:3626:x1870152993132800:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.867627:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1627 count 1 00010000:00010000:0.0:1783520935.867640:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025a07 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x65b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.867652:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000843c56ec/0xd5401cbabc02022c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x65b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ac0 expref: 680 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.867658:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x65b:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.867661:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x65b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.867665:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025a07 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x65b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.867688:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000843c56ec/0xd5401cbabc02022c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x65b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ac0 expref: 680 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.867696:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000843c56ec/0xd5401cbabc02022c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x65b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ac0 expref: 680 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.867708:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508059180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993132928:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.867759:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.867872:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.867976:0:17749:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002406016 00000100:00100000:2.0:1783520935.867982:0:17749:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002406016 00000100:00100000:2.0:1783520935.868022:0:17749:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98410 00000100:00100000:2.0:1783520935.868031:0:17749:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508058000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+28:251804:x1870153002406016:12345-192.168.204.33@tcp:36:dd.0 00010000:00010000:2.0:1783520935.868071:0:17749:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.868088:0:17749:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943512185c00/0xd5401cbabc0259e4 lrc: 0/0,0 mode: --/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 8 type: IBT gid 0 flags: 0x44a01000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34ea expref: 28 pid: 35279 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.868106:0:17749:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x72a:0x0].0x0 (ffff943515473e00) 00010000:00010000:2.0:1783520935.868110:0:17749:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.868147:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350805a300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993132160:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.868169:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0259f2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x658:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.868259:0:17749:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: mdt-lustre-MDT0000_UUID lock: ffff943512184800/0xd5401cbabc025a0e lrc: 3/0,1 mode: --/PW res: [0x200000402:0x72a:0x0].0x0 bits 0x0/0x0 rrc: 7 type: IBT flags: 0x40000000000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.868272:0:17749:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6e00/0xd5401cbabc025602 lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 7 type: IBT gid 0 flags: 0x40200000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34ce expref: 13 pid: 35280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.868279:0:17749:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: mdt-lustre-MDT0000_UUID lock: ffff943512184800/0xd5401cbabc025a0e lrc: 4/0,1 mode: --/PW res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 7 type: IBT flags: 0x50210000000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.868302:0:17749:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6e00/0xd5401cbabc025602 lrc: 3/0,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 7 type: IBT gid 0 flags: 0x50200000000020 nid: 192.168.204.33@tcp remote: 0xc275876b859f34ce expref: 13 pid: 35280 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.868311:0:17749:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6e00/0xd5401cbabc025602 lrc: 4/0,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 7 type: IBT gid 0 flags: 0x70200400000020 nid: 192.168.204.33@tcp remote: 0xc275876b859f34ce expref: 13 pid: 35280 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.868326:0:17749:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514f98e00 pname:cluuid:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000_UUID:17749:1870152993133056:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.868411:0:17749:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676d80 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.868435:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02597b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.868444:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02597b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.868453:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02597b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x64e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.868461:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02597b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x64e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.868483:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.868489:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.868506:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02597b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x64e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.868515:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.868519:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02597b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x64e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.868529:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02597b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x64e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.868543:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02597b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x64e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.868550:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02597b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x64e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.868578:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.868581:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5b9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+321:3627:x1870152993129472:12345-0@lo:6:osp-syn-1-0.0 Request processed in 9534us (10062us total) trans 4294975886 rc 0/0 00010000:00010000:0.0:1783520935.868582:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.868586:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.868594:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44379 00000004:00080000:1.0:1783520935.868603:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9d500/2, rc 0, transno 4294975886 00000100:00100000:1.0:1783520935.868619:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993129472:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.868621:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.868633:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.868648:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515777480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993132672:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.868660:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02599e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x651:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.868671:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02599e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x651:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.868678:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350805a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993133184:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.868679:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0259f9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x65a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.868702:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02599e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x651:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.868709:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02599e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x651:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.868718:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.868727:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.868741:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993133184 00000100:00100000:1.0:1783520935.868747:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993133184 00010000:00010000:0.0:1783520935.868758:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.868766:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02599e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x651:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.868774:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.868779:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02599e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x651:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.868785:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44393 00010000:00010000:0.0:1783520935.868792:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02599e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x651:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.868806:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02599e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x651:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.868813:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e007480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+322:3628:x1870152993130240:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7726us (8587us total) trans 4294975887 rc 0/0 00010000:00010000:0.0:1783520935.868820:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02599e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x651:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.868827:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e005500/2, rc 0, transno 4294975887 00000100:00100000:2.0:1783520935.868828:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44382 00000100:00100000:1.0:1783520935.868844:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e005500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993130240:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.868849:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.868853:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.868858:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.868863:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0259a5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x653:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.868870:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0259a5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x653:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.868876:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0259a5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x653:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.868881:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0259a5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x653:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.868897:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.868897:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515777480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993133312:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.868913:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.868919:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0259a5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x653:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.868928:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.868932:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0259a5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x653:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.868937:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.868947:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0259a5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x653:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.868960:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0259a5 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x653:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.868968:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0259a5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x653:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.868985:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993133312 00000100:00100000:2.0:1783520935.868986:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508059880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+322:3625:x1870152993130496:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7635us (7887us total) trans 4294975888 rc 0/0 00000100:00100000:1.0:1783520935.868990:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993133312 00000100:00100000:2.0:1783520935.868997:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44383 00000004:00080000:0.0:1783520935.869011:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516682d80/2, rc 0, transno 4294975888 00000100:00100000:2.0:1783520935.869018:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44394 00000100:00100000:2.0:1783520935.869027:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515777100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+322:3627:x1870152993133312:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.869030:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516682d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993130496:0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.869044:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1629 count 1 00000100:00100000:0.0:1783520935.869060:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350805b800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993132544:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.869061:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025a15 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x65d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.869065:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1783520935.869078:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515774380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993133440:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.869080:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b9c181ce/0xd5401cbabc02023a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x65d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0adc expref: 680 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.869082:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025a00 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x659:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.869105:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x65d:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.869112:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x65d:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.869113:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.869115:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.869120:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025a15 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x65d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.869120:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.869124:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.869148:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b9c181ce/0xd5401cbabc02023a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x65d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0adc expref: 680 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.869149:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993133440 00000100:00100000:1.0:1783520935.869154:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993133440 00000100:00100000:0.0:1783520935.869162:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.869167:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b9c181ce/0xd5401cbabc02023a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x65d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0adc expref: 680 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.869180:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943634573100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993133568:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.869184:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44395 00000100:00100000:1.0:1783520935.869196:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5bad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+323:3627:x1870152993133440:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.869209:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1630 count 1 00010000:00010000:1.0:1783520935.869223:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025a1c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x65e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.869225:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435126766c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.869237:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.869245:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000685267fc/0xd5401cbabc022b68 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x65e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3258 expref: 680 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.869253:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x65e:0x0].0x0 since=36s v=(0 0) 00010000:00010000:1.0:1783520935.869257:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x65e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.869262:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025a1c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x65e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.869285:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000685267fc/0xd5401cbabc022b68 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x65e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3258 expref: 680 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.869290:0:20415:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc020202 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x655:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a6c expref: 680 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.869322:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000685267fc/0xd5401cbabc022b68 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x65e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3258 expref: 680 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.869328:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517244e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993131392:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.869337:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.869344:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993133696:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.869355:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0259ba lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x655:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.869389:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0259ba lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x655:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.869396:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0259ba lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x655:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.869404:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0259ba lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x655:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.869411:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0259ba lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x655:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.869427:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.869437:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.869442:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0259ba lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x655:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.869453:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.869454:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.869460:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0259ba lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x655:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.869465:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511a4c600 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.869476:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0259ba lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x655:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.869484:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002406144 00000100:00100000:1.0:1783520935.869489:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002406144 00010000:00010000:2.0:1783520935.869490:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0259ba lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x655:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.869511:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0259ba lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x655:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.869521:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000019472967/0xd5401cbabc020210 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x658:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a96 expref: 680 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.869529:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.869533:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.869536:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.869545:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002406400 00000100:00100000:1.0:1783520935.869549:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002406400 00010000:00010000:1.0:1783520935.869563:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007c066493/0xd5401cbabc02021e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x65a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ab2 expref: 681 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.869570:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002406272 00000100:00100000:2.0:1783520935.869575:0:17749:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514f98e00 pname:cluuid:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000_UUID:17749:1870152993133056:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.869576:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002406272 00000100:00100000:1.0:1783520935.869583:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002406528 00000100:00100000:1.0:1783520935.869587:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002406528 00010000:00010000:2.0:1783520935.869597:0:17749:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: mdt-lustre-MDT0000_UUID lock: ffff943512184800/0xd5401cbabc025a0e lrc: 3/0,1 mode: --/PW res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 7 type: IBT flags: 0x40210000000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.869601:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000099edab59/0xd5401cbabc020217 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x659:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0aa4 expref: 683 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.869606:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000843c56ec/0xd5401cbabc02022c lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x65b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ac0 expref: 683 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.869626:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35981 00000100:00100000:2.0:1783520935.869635:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350805b800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+683:2390:x1870153002406144:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.869645:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.869650:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35982 00010000:00010000:2.0:1783520935.869655:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000019472967/0xd5401cbabc020210 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x658:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a96 expref: 683 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.869663:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35983 00000100:00100000:1.0:1783520935.869664:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350805b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+683:2388:x1870153002406400:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.869677:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508059f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+683:2387:x1870153002406272:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.869679:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.869683:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000019472967/0xd5401cbabc020210 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x658:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a96 expref: 683 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.869689:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007c066493/0xd5401cbabc02021e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x65a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ab2 expref: 683 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.869692:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.869703:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000099edab59/0xd5401cbabc020217 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x659:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0aa4 expref: 683 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.869722:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007c066493/0xd5401cbabc02021e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x65a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ab2 expref: 683 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.869730:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000099edab59/0xd5401cbabc020217 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x659:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0aa4 expref: 683 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.869732:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000019472967/0xd5401cbabc020210 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x658:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0a96 expref: 683 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.869743:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007c066493/0xd5401cbabc02021e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x65a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ab2 expref: 683 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.869762:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x658:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.869766:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x65a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.869766:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000099edab59/0xd5401cbabc020217 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x659:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0aa4 expref: 681 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.869768:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x658:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.869774:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x65a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.869779:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0259f2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x658:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.869784:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0259f9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.869796:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x659:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.869805:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x659:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.869807:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0259f2 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x658:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.869808:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0259f9 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x65a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.869813:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025a00 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x659:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.869838:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025a00 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x659:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.869844:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.869851:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.869862:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.869874:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350805b800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+680:2390:x1870153002406144:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 239us (716us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.869879:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025a00 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x659:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.869881:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350805b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+680:2388:x1870153002406400:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 220us (647us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.869889:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35981 00010000:00010000:0.0:1783520935.869899:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025a00 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x659:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.869901:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35982 00010000:00010000:0.0:1783520935.869913:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025a00 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x659:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.869920:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35984 00010000:00010000:0.0:1783520935.869929:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025a00 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x659:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.869934:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c2680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+678:2387:x1870153002406528:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.869942:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0259f9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.869951:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.869952:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.869953:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0259f9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.869962:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000843c56ec/0xd5401cbabc02022c lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x65b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ac0 expref: 678 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.869967:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0259f9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.869981:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000843c56ec/0xd5401cbabc02022c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x65b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ac0 expref: 678 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.869981:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0259f9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x65a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.870014:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520935.870018:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x65b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.870023:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x65b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.870025:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520935.870028:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025a07 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.870031:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025a00 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x659:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.870042:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025a07 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x65b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.870045:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.870052:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025a00 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x659:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.870054:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506228a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+324:3628:x1870152993132288:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3130us (3203us total) trans 4294975889 rc 0/0 00010000:00010000:2.0:1783520935.870066:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.870072:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44391 00010000:00010000:1.0:1783520935.870074:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025a00 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x659:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.870089:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025a00 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x659:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.870094:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c2680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+678:2387:x1870153002406528:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 162us (644us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.870103:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025a00 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x659:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.870110:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35984 00000100:00100000:0.0:1783520935.870120:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508059f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+678:2387:x1870153002406272:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 445us (789us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.870127:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35983 00010000:00010000:1.0:1783520935.870131:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.870136:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0259f9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x65a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.870142:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.870150:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbaa00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.870152:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0259f2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x658:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.870155:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0259f9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x65a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.870166:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0259f2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x658:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.870173:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0259f9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x65a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.870191:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0259f2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x658:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.870200:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0259f2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x658:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.870211:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0259f9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x65a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.870215:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.870224:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0259f9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x65a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.870228:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.870235:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0259f2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x658:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.870243:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.870246:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516682300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+324:3627:x1870152993131264:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7961us (8054us total) trans 4294975890 rc 0/0 00010000:00010000:1.0:1783520935.870247:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.870247:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.870252:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.870254:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0259f2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x658:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.870261:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44386 00010000:00010000:0.0:1783520935.870271:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0259f2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x658:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.870278:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516680e00/2, rc 0, transno 4294975890 00010000:00010000:0.0:1783520935.870285:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0259f2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x658:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.870291:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516680e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993131264:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.870301:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0259f2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x658:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.870322:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.870324:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.870326:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.870350:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993133824:0@lo:6:osp-syn-1-0.0 00000004:00080000:1.0:1783520935.870389:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350805a680/2, rc 0, transno 4294975889 00000100:00100000:1.0:1783520935.870399:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350805a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993132288:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.870409:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.870446:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1783520935.870454:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993133952:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.870486:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.870545:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993133824 00000100:00100000:1.0:1783520935.870546:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634571c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+324:3628:x1870152993132416:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3639us (3760us total) trans 4294975891 rc 0/0 00000100:00100000:0.0:1783520935.870551:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993133824 00000100:00100000:1.0:1783520935.870558:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44390 00000100:00100000:0.0:1783520935.870591:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993133952 00000100:00100000:0.0:1783520935.870596:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993133952 00000004:00080000:1.0:1783520935.870598:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350805aa00/2, rc 0, transno 4294975891 00000100:00100000:1.0:1783520935.870610:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350805aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993132416:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.870625:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44396 00000100:00100000:2.0:1783520935.870638:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44397 00000100:00100000:0.0:1783520935.870638:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+325:3625:x1870152993133824:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.870652:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+325:3625:x1870152993133952:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.870656:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1631 count 1 00000100:00100000:1.0:1783520935.870660:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993134080:0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.870666:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1632 count 1 00010000:00010000:0.0:1783520935.870674:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025a23 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x65f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.870682:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025a2a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x660:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.870689:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000081a739ae/0xd5401cbabc020248 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x65f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0af8 expref: 676 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.870701:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.870703:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b3c1d09a/0xd5401cbabc02024f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x660:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b06 expref: 676 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.870709:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x65f:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.870715:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x65f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.870722:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025a23 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x65f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.870737:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x660:0x0].0x0 since=37s v=(0 0) 00000100:00100000:1.0:1783520935.870740:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993134080 00010000:00010000:2.0:1783520935.870743:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x660:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.870745:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993134080 00010000:00010000:2.0:1783520935.870749:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025a2a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x660:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.870753:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000081a739ae/0xd5401cbabc020248 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x65f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0af8 expref: 676 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.870772:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44398 00010000:00010000:0.0:1783520935.870777:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000081a739ae/0xd5401cbabc020248 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x65f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0af8 expref: 676 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.870784:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+326:3627:x1870152993134080:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.870790:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c3800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993134208:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.870795:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b3c1d09a/0xd5401cbabc02024f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x660:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b06 expref: 676 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.870801:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1633 count 1 00010000:00010000:2.0:1783520935.870816:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b3c1d09a/0xd5401cbabc02024f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x660:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b06 expref: 676 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.870817:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025a31 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x661:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.870825:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470b40 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.870843:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943634571500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993134336:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.870844:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008b96fef1/0xd5401cbabc020256 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x661:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b14 expref: 676 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.870856:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x661:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.870859:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x661:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.870863:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025a31 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x661:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.870874:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676c00 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.870887:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008b96fef1/0xd5401cbabc020256 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x661:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b14 expref: 676 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.870893:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350805ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+327:3626:x1870152993133184:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.870905:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008b96fef1/0xd5401cbabc020256 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x661:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b14 expref: 676 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.870909:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1628 count 1 00000100:00100000:1.0:1783520935.870921:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993134464:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.870923:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025a38 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x65c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.870932:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634571f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+327:3626:x1870152993132032:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4877us (5093us total) trans 4294975892 rc 0/0 00010000:00010000:2.0:1783520935.870937:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000aa87ae28/0xd5401cbabc020233 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x65c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ace expref: 676 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.870945:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44389 00010000:00010000:2.0:1783520935.870947:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x65c:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.870951:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x65c:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.870953:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511a4c6c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.870954:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025a38 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x65c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.870972:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000aa87ae28/0xd5401cbabc020233 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x65c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ace expref: 676 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.870978:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943508058700/2, rc 0, transno 4294975892 00010000:00010000:2.0:1783520935.870980:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000aa87ae28/0xd5401cbabc020233 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x65c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ace expref: 676 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.870990:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943634571180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993134592:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.871000:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508058700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993132032:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.871014:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676d80 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.871040:0:35276:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000843c56ec/0xd5401cbabc02022c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x65b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ac0 expref: 676 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.871050:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508059180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993132928:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.871062:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.871065:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025a07 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.871074:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025a07 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.871082:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025a07 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.871096:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025a07 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.871105:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025a07 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.871125:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.871139:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.871145:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025a07 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.871154:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbebe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.871158:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025a07 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.871167:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025a07 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.871181:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025a07 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.871189:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025a07 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.871195:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515776680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+327:3626:x1870152993132800:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3582us (3657us total) trans 4294975893 rc 0/0 00000100:00100000:0.0:1783520935.871204:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44392 00010000:00010000:2.0:1783520935.871209:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.871213:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.871215:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.871240:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943634573100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993133568:192.168.204.33@tcp:104: 00000004:00080000:0.0:1783520935.871243:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943508058a80/2, rc 0, transno 4294975893 00000100:00100000:0.0:1783520935.871252:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508058a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993132800:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.871257:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025a15 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x65d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.871326:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993133696:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.871335:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.871341:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025a1c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x65e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.871438:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.871463:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002406656 00000100:00100000:2.0:1783520935.871469:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002406784 00000100:00100000:0.0:1783520935.871469:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002406656 00000100:00100000:2.0:1783520935.871475:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002406784 00010000:00010000:0.0:1783520935.871492:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6e00/0xd5401cbabc025602 lrc: 4/0,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 7 type: IBT gid 0 flags: 0x60200400000020 nid: 192.168.204.33@tcp remote: 0xc275876b859f34ce expref: 14 pid: 35280 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.871510:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b9c181ce/0xd5401cbabc02023a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x65d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0adc expref: 676 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.871527:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002406912 00000100:00100000:0.0:1783520935.871531:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002406912 00010000:00010000:0.0:1783520935.871547:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000685267fc/0xd5401cbabc022b68 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x65e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3258 expref: 677 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.871549:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35985 00000100:00100000:2.0:1783520935.871564:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634570700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:25342ed3-dd17-4439-8398-d8002e9f9f99+14:2388:x1870153002406656:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.871578:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.871587:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6e00/0xd5401cbabc025602 lrc: 4/0,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 8 type: IBT gid 0 flags: 0x60200400000020 nid: 192.168.204.33@tcp remote: 0xc275876b859f34ce expref: 14 pid: 35280 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.871587:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35986 00000100:00100000:0.0:1783520935.871600:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943618f35c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+677:2387:x1870153002406784:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.871604:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6e00/0xd5401cbabc025602 lrc: 3/0,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 8 type: IBT gid 0 flags: 0x50200400000020 nid: 192.168.204.33@tcp remote: 0xc275876b859f34ce expref: 14 pid: 35280 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.871610:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.871617:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b9c181ce/0xd5401cbabc02023a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x65d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0adc expref: 677 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.871629:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b9c181ce/0xd5401cbabc02023a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x65d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0adc expref: 677 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.871630:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943514fa6e00/0xd5401cbabc025602 lrc: 0/0,0 mode: --/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 8 type: IBT gid 0 flags: 0x44a01400000020 nid: 192.168.204.33@tcp remote: 0xc275876b859f34ce expref: 14 pid: 35280 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.871656:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b9c181ce/0xd5401cbabc02023a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x65d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0adc expref: 677 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.871661:0:8424:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x72a:0x0].0x0 (ffff943515473e00) 00010000:00010000:2.0:1783520935.871669:0:8424:0:(ldlm_inodebits.c:93:ldlm_reprocess_inodebits_queue()) ### Reprocessing lock from queue 1 ns: mdt-lustre-MDT0000_UUID lock: ffff943512184800/0xd5401cbabc025a0e lrc: 3/0,1 mode: --/PW res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 7 type: IBT flags: 0x40210000000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.871675:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x65d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.871679:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: mdt-lustre-MDT0000_UUID lock: ffff943512184800/0xd5401cbabc025a0e lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 7 type: IBT flags: 0x40290000000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.871680:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x65d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.871690:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025a15 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.871692:0:8424:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943512184800/0xd5401cbabc025a0e lrc: 4/0,1 mode: PW/PW res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 7 type: IBT flags: 0x40290000000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.871711:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025a15 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x65d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.871725:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.871736:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.871751:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634570700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:25342ed3-dd17-4439-8398-d8002e9f9f99+13:2388:x1870153002406656:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 191us (691us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.871761:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943618f35c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+676:2387:x1870153002406784:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 162us (424us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.871762:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35985 00000100:00100000:0.0:1783520935.871774:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35986 00000100:00100000:2.0:1783520935.871786:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35987 00000100:00100000:2.0:1783520935.871798:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634573100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+675:2388:x1870153002406912:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.871808:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.871814:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000685267fc/0xd5401cbabc022b68 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x65e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3258 expref: 675 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.871826:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000685267fc/0xd5401cbabc022b68 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x65e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3258 expref: 675 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.871841:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000685267fc/0xd5401cbabc022b68 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x65e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3258 expref: 675 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.871859:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x65e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.871862:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x65e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.871867:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025a1c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.871874:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025a1c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x65e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.871897:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.871909:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025a1c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.871917:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634573100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+674:2388:x1870153002406912:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 121us (478us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.871924:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025a1c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.871930:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35987 00010000:00010000:0.0:1783520935.871940:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025a1c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.871948:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025a1c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x65e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.871958:0:17749:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: mdt-lustre-MDT0000_UUID lock: ffff943512184800/0xd5401cbabc025a0e lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x40210000000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.871964:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.871967:0:17749:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943512184800/0xd5401cbabc025a0e lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x40210000000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00000001:00080000:2.0:1783520935.872049:0:17749:0:(tgt_lastrcvd.c:2117:tgt_txn_stop_cb()) More than one transaction 8589957899 00000100:00100000:0.0:1783520935.872069:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5bad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+327:3627:x1870152993133440:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2876us (2959us total) trans 4294975894 rc 0/0 00000100:00100000:0.0:1783520935.872078:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44395 00000004:00010000:2.0:1783520935.872092:0:17749:0:(mdt_handler.c:4478:mdt_save_lock()) ### save lock request 00000000f18e1ca2 reply state 0000000002ccfcdd transno 8589957900 ns: mdt-lustre-MDT0000_UUID lock: ffff943512184800/0xd5401cbabc025a0e lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x40210000000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.872107:0:17749:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943512184800/0xd5401cbabc025a0e lrc: 3/0,1 mode: TXN/TXN res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x50210000000000 pid: 17749 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.872114:0:17749:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x72a:0x0].0x0 (ffff943515473e00) 00010000:00010000:2.0:1783520935.872118:0:17749:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00000100:00100000:2.0:1783520935.872121:0:17749:0:(service.c:179:ptlrpc_save_lock()) nlocks 0 00000100:00100000:2.0:1783520935.872191:0:17749:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508058000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+30:251804:x1870153002406016:12345-192.168.204.33@tcp:36:dd.0 Request processed in 4160us (4319us total) trans 8589957900 rc 0/0 00000100:00100000:2.0:1783520935.872200:0:17749:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98410 00000020:00080000:2.0:1783520935.872205:0:17749:0:(tgt_main.c:784:tgt_ses_key_exit()) total 2 transactions per RPC 00010000:00010000:2.0:1783520935.872227:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.872233:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025a1c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x65e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.872243:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbbc00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.872249:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025a1c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x65e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.872259:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025a1c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x65e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.872276:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025a1c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x65e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.872286:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025a1c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x65e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.872310:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.872312:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.872318:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.872339:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025a15 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.872352:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025a15 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.872389:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025a15 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.872402:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025a15 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x65d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.872424:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.872475:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e007100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993134848:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.872524:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:1.0:1783520935.872563:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515774380/2, rc 0, transno 4294975894 00000100:00100000:2.0:1783520935.872574:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515777100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+327:3627:x1870152993133312:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3543us (3634us total) trans 4294975895 rc 0/0 00000100:00100000:1.0:1783520935.872594:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515774380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993133440:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.872595:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44394 00000100:00100000:0.0:1783520935.872595:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993134848 00000100:00100000:0.0:1783520935.872606:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993134848 00000004:00080000:1.0:1783520935.872634:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515777480/2, rc 0, transno 4294975895 00000100:00100000:1.0:1783520935.872645:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515777480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993133312:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.872656:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44399 00000100:00100000:1.0:1783520935.872665:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1783520935.872675:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+327:3625:x1870152993134848:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.872691:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993134720:0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.872695:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1635 count 1 00010000:00010000:0.0:1783520935.872717:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025a3f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x663:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.872729:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009fd2ede6/0xd5401cbabc020264 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x663:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b30 expref: 673 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.872735:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x663:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.872738:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x663:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.872742:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025a3f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x663:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.872772:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009fd2ede6/0xd5401cbabc020264 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x663:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b30 expref: 673 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.872773:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.872793:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009fd2ede6/0xd5401cbabc020264 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x663:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b30 expref: 673 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.872807:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c2d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993134976:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.872856:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.872868:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025a15 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x65d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.872870:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470900 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.872885:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.872892:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025a15 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x65d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.872897:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993134720 00010000:00010000:1.0:1783520935.872904:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025a15 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x65d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.872905:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993134720 00010000:00010000:1.0:1783520935.872934:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025a15 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x65d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.872944:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025a15 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x65d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.872944:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44400 00000100:00100000:0.0:1783520935.872960:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e006d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+328:3627:x1870152993134720:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.872974:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1634 count 1 00010000:00010000:1.0:1783520935.872988:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.872989:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025a46 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x662:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.872994:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.873007:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.873008:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008afce796/0xd5401cbabc02025d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x662:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b22 expref: 673 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.873023:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x662:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.873027:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x662:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.873031:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025a46 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x662:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.873060:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008afce796/0xd5401cbabc02025d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x662:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b22 expref: 673 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.873076:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008afce796/0xd5401cbabc02025d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x662:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b22 expref: 673 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.873090:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e005c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993135104:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.873092:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993135232:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.873151:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.873157:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470e40 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.873177:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993135104 00000100:00100000:0.0:1783520935.873181:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993135104 00000100:00100000:0.0:1783520935.873231:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44401 00000100:00100000:1.0:1783520935.873236:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e004e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993135360:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.873248:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e98e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+329:3626:x1870152993135104:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.873262:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1636 count 1 00010000:00010000:0.0:1783520935.873275:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025a4d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x664:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.873291:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.873300:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007d644440/0xd5401cbabc02026b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x664:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b3e expref: 673 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.873312:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x664:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.873314:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x664:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.873318:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025a4d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x664:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.873344:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007d644440/0xd5401cbabc02026b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x664:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b3e expref: 673 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.873354:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007d644440/0xd5401cbabc02026b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x664:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b3e expref: 673 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.873448:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993135360 00000100:00100000:1.0:1783520935.873456:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993135360 00000100:00100000:0.0:1783520935.873492:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c0e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993135488:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.873532:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44402 00000100:00100000:1.0:1783520935.873548:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+330:3627:x1870152993135360:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.873561:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00002000:00080000:1.0:1783520935.873575:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1637 count 1 00010000:00010000:1.0:1783520935.873600:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc025a54 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x665:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.873618:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000033600ef7/0xd5401cbabc020272 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x665:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b4c expref: 673 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.873627:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x665:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.873631:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x665:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.873636:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc025a54 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x665:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.873669:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000033600ef7/0xd5401cbabc020272 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x665:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b4c expref: 673 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.873682:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000033600ef7/0xd5401cbabc020272 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x665:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b4c expref: 673 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.873701:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e99f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993135616:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.873761:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511a4c600 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.875174:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c3800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993134208:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.875211:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025a23 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x65f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.876472:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943634571500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993134336:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.876506:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025a2a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x660:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.876700:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c2d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993134976:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.876703:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943634571180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993134592:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.876744:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025a38 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x65c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.876755:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025a3f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x663:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.876857:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993135232:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.876885:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025a46 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x662:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.876966:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.876989:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002407552 00000100:00100000:0.0:1783520935.876996:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002407552 00010000:00010000:0.0:1783520935.877049:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000aa87ae28/0xd5401cbabc020233 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x65c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ace expref: 674 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.877101:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35988 00000100:00100000:0.0:1783520935.877121:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+674:2390:x1870153002407552:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.877141:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.877148:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000aa87ae28/0xd5401cbabc020233 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x65c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ace expref: 674 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.877165:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000aa87ae28/0xd5401cbabc020233 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x65c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ace expref: 674 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.877204:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000aa87ae28/0xd5401cbabc020233 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x65c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ace expref: 674 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.877208:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993134464:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.877235:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x65c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.877241:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x65c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.877248:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025a38 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.877249:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025a31 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x661:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.877280:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025a38 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x65c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.877307:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e99f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993135616:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.877325:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.877338:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc025a54 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x665:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.877394:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025a38 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.877415:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025a38 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.877429:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025a38 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.877437:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025a38 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x65c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.877462:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.877473:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.877513:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+673:2390:x1870153002407552:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 392us (542us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.877516:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002407936 00000100:00100000:2.0:1783520935.877527:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002407936 00000100:00100000:0.0:1783520935.877533:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35988 00010000:00010000:2.0:1783520935.877565:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008afce796/0xd5401cbabc02025d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x662:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b22 expref: 673 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.877580:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.877597:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025a38 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x65c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.877611:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.877619:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025a38 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x65c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.877632:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35989 00010000:00010000:0.0:1783520935.877647:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025a38 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x65c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.877649:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.877658:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e99f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+673:2390:x1870153002407936:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.877688:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.877691:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025a38 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x65c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.877705:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.877706:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008afce796/0xd5401cbabc02025d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x662:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b22 expref: 673 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.877710:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025a38 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x65c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.877743:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.877771:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008afce796/0xd5401cbabc02025d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x662:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b22 expref: 673 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.877793:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.877801:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.877806:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.877810:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.877822:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008afce796/0xd5401cbabc02025d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x662:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b22 expref: 673 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.877851:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x662:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.877861:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c0e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993135488:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.877863:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.877864:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x662:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.877884:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025a46 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x662:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.877898:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025a46 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x662:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.877900:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025a4d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x664:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.877918:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.877955:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002407168 00010000:00010000:2.0:1783520935.877964:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.877964:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002407168 00000100:00100000:1.0:1783520935.877982:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.877995:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025a46 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x662:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.878007:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025a46 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x662:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.878020:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025a46 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x662:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.878029:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025a46 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x662:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.878040:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002407040 00000100:00100000:2.0:1783520935.878045:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e99f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+673:2390:x1870153002407936:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 391us (578us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.878048:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002407040 00010000:00010000:0.0:1783520935.878054:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520935.878065:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35989 00010000:00010000:1.0:1783520935.878084:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000081a739ae/0xd5401cbabc020248 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x65f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0af8 expref: 674 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.878091:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002407296 00000100:00100000:2.0:1783520935.878099:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002407296 00000100:00100000:1.0:1783520935.878121:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002407424 00010000:00010000:2.0:1783520935.878127:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008b96fef1/0xd5401cbabc020256 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x661:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b14 expref: 674 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.878130:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002407424 00000100:00100000:2.0:1783520935.878157:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002407808 00000100:00100000:2.0:1783520935.878168:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002407808 00010000:00010000:1.0:1783520935.878168:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b3c1d09a/0xd5401cbabc02024f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x660:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b06 expref: 675 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.878194:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007d644440/0xd5401cbabc02026b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x664:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b3e expref: 676 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.878200:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002407680 00000100:00100000:1.0:1783520935.878206:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002407680 00010000:00010000:1.0:1783520935.878233:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009fd2ede6/0xd5401cbabc020264 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x663:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b30 expref: 677 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.878245:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35990 00000100:00100000:1.0:1783520935.878279:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35991 00000100:00100000:2.0:1783520935.878281:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+677:2387:x1870153002407040:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.878301:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9a680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+677:2388:x1870153002407296:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.878306:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.878316:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.878319:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000081a739ae/0xd5401cbabc020248 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x65f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0af8 expref: 677 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.878321:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e006d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+331:3627:x1870152993134720:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5359us (5553us total) trans 4294975896 rc 0/0 00010000:00010000:1.0:1783520935.878330:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008b96fef1/0xd5401cbabc020256 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x661:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b14 expref: 677 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.878404:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44400 00010000:00010000:2.0:1783520935.878433:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000081a739ae/0xd5401cbabc020248 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x65f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0af8 expref: 677 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.878444:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008b96fef1/0xd5401cbabc020256 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x661:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b14 expref: 677 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.878464:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000081a739ae/0xd5401cbabc020248 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x65f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0af8 expref: 677 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.878474:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008b96fef1/0xd5401cbabc020256 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x661:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b14 expref: 677 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.878478:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9df80/2, rc 0, transno 4294975896 00010000:00010000:2.0:1783520935.878486:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x65f:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.878513:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993134720:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.878514:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x65f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.878527:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x661:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.878528:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025a23 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.878547:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x661:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.878549:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025a23 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x65f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.878557:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025a31 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x661:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.878575:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025a31 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x661:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.878583:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.878622:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+675:2387:x1870153002407040:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 362us (924us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.878631:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35990 00000100:00100000:0.0:1783520935.878645:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993135744:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.878657:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35992 00000100:00100000:2.0:1783520935.878673:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943618f37850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+674:2387:x1870153002407424:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.878689:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.878696:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.878711:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b3c1d09a/0xd5401cbabc02024f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x660:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b06 expref: 674 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.878734:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b3c1d09a/0xd5401cbabc02024f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x660:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b06 expref: 674 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.878743:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35993 00000100:00100000:1.0:1783520935.878767:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9b800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+674:2388:x1870153002407808:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.878776:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b3c1d09a/0xd5401cbabc02024f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x660:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b06 expref: 674 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.878784:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.878799:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007d644440/0xd5401cbabc02026b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x664:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b3e expref: 673 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.878806:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x660:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.878806:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025a23 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.878820:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007d644440/0xd5401cbabc02026b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x664:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b3e expref: 673 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.878823:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x660:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.878854:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025a23 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.878856:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025a2a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x660:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.878874:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007d644440/0xd5401cbabc02026b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x664:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b3e expref: 673 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.878908:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025a23 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.878914:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025a2a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x660:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.878926:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x664:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.878946:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x664:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.878951:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025a23 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.878962:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025a4d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x664:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.878990:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025a4d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x664:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.878994:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.878997:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520935.879022:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.879046:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943618f37850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+672:2387:x1870153002407424:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 378us (1246us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.879058:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35992 00000100:00100000:1.0:1783520935.879067:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9b800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+672:2388:x1870153002407808:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 302us (1209us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.879081:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35993 00000100:00100000:2.0:1783520935.879097:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35994 00000100:00100000:2.0:1783520935.879112:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e98700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+670:2387:x1870153002407680:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.879126:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.879133:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.879136:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009fd2ede6/0xd5401cbabc020264 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x663:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b30 expref: 670 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.879148:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+331:3625:x1870152993133824:12345-0@lo:6:osp-syn-1-0.0 Request processed in 8507us (8733us total) trans 4294975897 rc 0/0 00000100:00100000:0.0:1783520935.879167:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44396 00010000:00010000:2.0:1783520935.879174:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009fd2ede6/0xd5401cbabc020264 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x663:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b30 expref: 670 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.879182:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c5b8380/2, rc 0, transno 4294975897 00000100:00100000:0.0:1783520935.879199:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002408064 00000100:00100000:1.0:1783520935.879205:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993133824:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.879206:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002408064 00010000:00010000:2.0:1783520935.879207:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009fd2ede6/0xd5401cbabc020264 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x663:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b30 expref: 670 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.879241:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x663:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.879245:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x663:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.879252:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025a3f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x663:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.879261:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025a3f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x663:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.879263:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993135744 00000100:00100000:0.0:1783520935.879272:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993135744 00000100:00100000:1.0:1783520935.879290:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1783520935.879306:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993135872:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.879308:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.879318:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 44403 00000100:00100000:0.0:1783520935.879334:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e98380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+670:251804:x1870153002408064:12345-192.168.204.33@tcp:101:dd.0 00000100:00100000:2.0:1783520935.879351:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e98700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+670:2387:x1870153002407680:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 240us (1442us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.879352:0:178496:0:(ldlm_lockd.c:1265:ldlm_handle_enqueue()) ### server-side enqueue handler START 00000100:00100000:2.0:1783520935.879399:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35994 00010000:00010000:0.0:1783520935.879414:0:178496:0:(ldlm_lockd.c:1343:ldlm_handle_enqueue()) ### server-side enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000000c90bcf0/0xd5401cbabc025a5b lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0xc275876b859f34f1 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.879434:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520935.879442:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025a46 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x662:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00010000:0.0:1783520935.879448:0:178496:0:(ofd_lvb.c:151:ofd_lvbo_init()) res: [0x2c0000401:0x8f7:0x0] initial LVB size: 262144, mtime: 0x6a4e5ea8, atime: 0x0, ctime: 0x6a4e5ea8, blocks: 0x100 00010000:00010000:2.0:1783520935.879464:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.879473:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025a46 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x662:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.879476:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f7:0x0].0x0 since=4130s v=(0 0) 00010000:00010000:0.0:1783520935.879507:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.879513:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025a46 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x662:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.879519:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000c90bcf0/0xd5401cbabc025a5b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34f1 expref: 670 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.879551:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025a46 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x662:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.879561:0:178496:0:(ldlm_lockd.c:1513:ldlm_handle_enqueue()) ### server-side enqueue handler, sending reply (err=0, rc=0) ns: filter-lustre-OST0001_UUID lock: 000000000c90bcf0/0xd5401cbabc025a5b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34f1 expref: 670 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.879566:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025a46 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x662:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.879599:0:178496:0:(ldlm_lockd.c:1592:ldlm_handle_enqueue()) ### server-side enqueue handler END (lock ffff94351cbeae00, rc 0) 00010000:00010000:2.0:1783520935.879614:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.879619:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025a23 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.879627:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.879632:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025a23 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.879642:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e98380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+670:251804:x1870153002408064:12345-192.168.204.33@tcp:101:dd.0 Request processed in 308us (1664us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.879656:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025a23 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.879666:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 44403 00010000:00010000:2.0:1783520935.879685:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025a23 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.879697:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025a23 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.879708:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44404 00000100:00100000:0.0:1783520935.879727:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+331:3628:x1870152993135744:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.879736:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.879740:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.879743:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00002000:00080000:0.0:1783520935.879751:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1638 count 1 00010000:00010000:2.0:1783520935.879752:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.879774:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025a62 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x666:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.879775:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025a2a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.879809:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025a2a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.879816:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000038a0c49e/0xd5401cbabc022a57 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x666:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3147 expref: 669 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.879847:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025a2a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.879848:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x666:0x0].0x0 since=36s v=(0 0) 00010000:00010000:0.0:1783520935.879862:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x666:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.879879:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025a62 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x666:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.879881:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025a2a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.879914:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.879921:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000038a0c49e/0xd5401cbabc022a57 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x666:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3147 expref: 669 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.879938:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000038a0c49e/0xd5401cbabc022a57 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x666:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3147 expref: 669 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.879953:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c3800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993136000:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.880022:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470900 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.880056:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025a3f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x663:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.880066:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025a3f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x663:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.880075:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025a3f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x663:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.880083:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025a3f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x663:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.880089:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+332:3625:x1870152993133952:12345-0@lo:6:osp-syn-1-0.0 Request processed in 9436us (9598us total) trans 4294975898 rc 0/0 00000100:00100000:2.0:1783520935.880118:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44397 00010000:00010000:0.0:1783520935.880130:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:0.0:1783520935.880266:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+333:3625:x1870152993134848:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7597us (7737us total) trans 4294975899 rc 0/0 00000100:00100000:0.0:1783520935.880282:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44399 00000100:00100000:2.0:1783520935.880300:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350805ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+332:3626:x1870152993133184:12345-0@lo:6:osp-syn-1-0.0 Request processed in 11487us (11583us total) trans 4294975900 rc 0/0 00000100:00100000:2.0:1783520935.880310:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44393 00010000:00010000:0.0:1783520935.880326:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520935.880336:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025a2a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.880346:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.880351:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025a2a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.880395:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025a2a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.880406:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.880419:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025a2a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.880429:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025a2a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.880472:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000004:00080000:1.0:1783520935.880477:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350805a300/2, rc 0, transno 4294975900 00010000:00010000:0.0:1783520935.880478:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025a3f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x663:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.880489:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.880503:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025a3f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x663:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.880507:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350805a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993133184:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.880522:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025a3f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x663:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.880537:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025a3f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x663:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.880538:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.880549:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025a3f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x663:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.880579:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e007100/2, rc 0, transno 4294975899 00010000:00010000:0.0:1783520935.880589:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.880590:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e007100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993134848:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.880595:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.880603:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520935.880620:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c5b9180/2, rc 0, transno 4294975898 00000100:00100000:1.0:1783520935.880626:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993133952:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.880644:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000033600ef7/0xd5401cbabc020272 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x665:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b4c expref: 669 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.880688:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35995 00000100:00100000:0.0:1783520935.880708:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e99880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+669:2388:x1870153002407168:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.880738:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993136256:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.880755:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1783520935.880767:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e99c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993136128:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.880799:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.880825:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993135872 00000100:00100000:1.0:1783520935.880831:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993135872 00000100:00100000:0.0:1783520935.880848:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1783520935.880860:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993136384:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.880866:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993136256 00000100:00100000:1.0:1783520935.880870:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993136256 00000100:00100000:1.0:1783520935.880896:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44405 00000100:00100000:1.0:1783520935.880914:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+333:3626:x1870152993135872:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.880914:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.880926:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44406 00002000:00080000:1.0:1783520935.880936:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1639 count 1 00000100:00100000:2.0:1783520935.880949:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+333:3627:x1870152993136256:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.880962:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025a69 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x667:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.880986:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00080000:2.0:1783520935.880988:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1641 count 1 00010000:00010000:1.0:1783520935.880988:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ee012eaf/0xd5401cbabc020287 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x667:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b68 expref: 669 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.881008:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x667:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.881012:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x667:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.881017:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025a69 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x667:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.881024:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025a70 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x669:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.881043:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993136384 00010000:00010000:2.0:1783520935.881048:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b76e260d/0xd5401cbabc02028e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x669:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b84 expref: 669 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.881051:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993136384 00010000:00010000:1.0:1783520935.881054:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ee012eaf/0xd5401cbabc020287 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x667:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b68 expref: 669 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.881083:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993136128 00000100:00100000:0.0:1783520935.881091:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993136128 00010000:00010000:1.0:1783520935.881100:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ee012eaf/0xd5401cbabc020287 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x667:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b68 expref: 669 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.881102:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x669:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.881124:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x669:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.881130:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44407 00000100:00100000:1.0:1783520935.881136:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e98a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993136512:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.881139:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025a70 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x669:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.881152:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5b8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+335:3627:x1870152993136384:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.881180:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1642 count 1 00010000:00010000:0.0:1783520935.881202:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025a77 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x66a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.881204:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b76e260d/0xd5401cbabc02028e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x669:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b84 expref: 669 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.881211:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511a4c600 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.881232:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e4cb304f/0xd5401cbabc020295 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x66a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b92 expref: 669 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.881242:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x66a:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.881247:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x66a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.881253:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025a77 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x66a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.881284:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e4cb304f/0xd5401cbabc020295 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x66a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b92 expref: 669 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.881298:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b76e260d/0xd5401cbabc02028e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x669:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b84 expref: 669 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.881304:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e4cb304f/0xd5401cbabc020295 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x66a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b92 expref: 669 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.881322:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025a31 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.881354:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025a31 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.881389:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350805b800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993136768:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.881393:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025a31 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.881395:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943634571180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993136640:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.881423:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025a31 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.881445:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.881457:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470b40 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.881488:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44408 00000100:00100000:2.0:1783520935.881492:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676540 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.881513:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634444450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+335:3625:x1870152993136128:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.881531:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00002000:00080000:0.0:1783520935.881534:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1640 count 1 00010000:00010000:2.0:1783520935.881540:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025a31 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.881553:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025a7e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x668:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.881554:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.881563:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025a31 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.881593:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a8381c9c/0xd5401cbabc020280 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x668:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b76 expref: 669 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.881595:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025a31 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.881607:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+336:3627:x1870152993134080:12345-0@lo:6:osp-syn-1-0.0 Request processed in 10822us (10905us total) trans 4294975901 rc 0/0 00010000:00010000:0.0:1783520935.881638:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x668:0x0].0x0 since=37s v=(0 0) 00000100:00100000:1.0:1783520935.881640:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44398 00010000:00010000:0.0:1783520935.881652:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x668:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.881658:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025a7e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x668:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.881663:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025a31 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.881678:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025a31 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.881690:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a8381c9c/0xd5401cbabc020280 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x668:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b76 expref: 669 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.881700:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c5b8e00/2, rc 0, transno 4294975901 00010000:00010000:0.0:1783520935.881707:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a8381c9c/0xd5401cbabc020280 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x668:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b76 expref: 669 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.881716:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.881717:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993134080:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.881724:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.881725:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350805b100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993136896:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.881736:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.881779:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470e40 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.881804:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993137024:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.881851:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9a680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+669:2388:x1870153002407296:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 3557us (4115us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.881855:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.881868:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35991 00000100:00100000:1.0:1783520935.881935:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993137024 00000100:00100000:1.0:1783520935.881941:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993137024 00010000:00010000:0.0:1783520935.881942:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.881956:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000033600ef7/0xd5401cbabc020272 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x665:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b4c expref: 668 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.881985:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000033600ef7/0xd5401cbabc020272 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x665:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b4c expref: 668 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.881997:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44409 00000100:00100000:1.0:1783520935.882006:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508059f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+336:3625:x1870152993137024:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.882009:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000033600ef7/0xd5401cbabc020272 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x665:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b4c expref: 668 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.882022:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1643 count 1 00010000:00010000:0.0:1783520935.882028:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x665:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.882033:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x665:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.882037:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025a85 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x66b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.882038:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc025a54 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x665:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.882061:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc025a54 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x665:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.882064:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000dfec228a/0xd5401cbabc02029c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x66b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ba0 expref: 667 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.882079:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x66b:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.882083:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x66b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.882087:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025a85 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x66b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.882093:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.882119:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000dfec228a/0xd5401cbabc02029c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x66b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ba0 expref: 667 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.882124:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e99880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+667:2388:x1870153002407168:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 1424us (4478us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.882141:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000dfec228a/0xd5401cbabc02029c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x66b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ba0 expref: 667 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.882142:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35995 00000100:00100000:1.0:1783520935.882170:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba8380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993137152:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.882214:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511a4c6c0 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.882240:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc025a54 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x665:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.882247:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c3800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993136000:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.882259:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc025a54 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x665:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.882271:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc025a54 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x665:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.882279:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc025a54 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x665:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.882281:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025a62 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x666:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.882308:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.882321:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.882346:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520935.882349:0:20097:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002408192 00010000:00010000:2.0:1783520935.882354:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc025a54 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x665:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.882411:0:20097:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002408192 00010000:00010000:2.0:1783520935.882418:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.882425:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc025a54 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x665:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.882436:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc025a54 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x665:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00010000:0.0:1783520935.882445:0:20097:0:(ofd_dev.c:2825:ofd_punch_hpreq_check()) lustre-OST0001: refresh locks: 11811161089/2295 (65536->18446744073709551615) 00010000:00010000:2.0:1783520935.882452:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc025a54 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x665:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00100000:0.0:1783520935.882453:0:20097:0:(ofd_dev.c:2565:ofd_prolong_extent_locks()) Prolong locks for req ffff943634571500 with x1870153002408192 ext(65536->18446744073709551615) 00002000:00010000:0.0:1783520935.882468:0:20097:0:(ofd_dev.c:2830:ofd_punch_hpreq_check()) lustre-OST0001: refreshed 0 locks timeout for req ffff943634571500. 00010000:00010000:2.0:1783520935.882469:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e6072bc3/0xd5401cbabc025a54 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x665:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.882509:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+337:3627:x1870152993135360:12345-0@lo:6:osp-syn-1-0.0 Request processed in 8951us (9216us total) trans 4294975902 rc 0/0 00010000:00010000:2.0:1783520935.882519:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.882525:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44402 00010000:00010000:2.0:1783520935.882526:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.882531:0:20097:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 34736 00010000:00010000:2.0:1783520935.882534:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.882552:0:20097:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634571500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_io00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+667:2389:x1870153002408192:12345-192.168.204.33@tcp:10:dd.0 00000004:00080000:1.0:1783520935.882585:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e004e00/2, rc 0, transno 4294975902 00000100:00100000:1.0:1783520935.882600:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e004e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993135360:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.882645:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.882675:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993137280:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.882684:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002408320 00000100:00100000:2.0:1783520935.882694:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002408320 00000100:00100000:1.0:1783520935.882719:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.882727:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000038a0c49e/0xd5401cbabc022a57 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x666:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3147 expref: 668 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.882779:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993137280 00000100:00100000:2.0:1783520935.882786:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35996 00000100:00100000:1.0:1783520935.882786:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993137280 00000100:00100000:2.0:1783520935.882802:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634573100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+668:2388:x1870153002408320:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.882816:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.882824:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000038a0c49e/0xd5401cbabc022a57 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x666:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3147 expref: 668 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.882829:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44410 00000100:00100000:1.0:1783520935.882849:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cbab100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+337:3626:x1870152993137280:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.882856:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000038a0c49e/0xd5401cbabc022a57 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x666:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3147 expref: 668 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.882870:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1644 count 1 00010000:00010000:2.0:1783520935.882880:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000038a0c49e/0xd5401cbabc022a57 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x666:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3147 expref: 669 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.882888:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025a8c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x66c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.882895:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x666:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.882903:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x666:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.882906:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004f01fcd0/0xd5401cbabc0202a3 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x66c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bae expref: 668 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.882911:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025a62 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x666:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.882933:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x66c:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.882946:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x66c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.882950:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025a62 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x666:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.882952:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025a8c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x66c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.883017:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004f01fcd0/0xd5401cbabc0202a3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x66c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bae expref: 668 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.883018:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.883036:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004f01fcd0/0xd5401cbabc0202a3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x66c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bae expref: 668 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.883050:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cbaa680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993137408:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.883062:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634573100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+668:2388:x1870153002408320:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 259us (419us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.883072:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35996 00002000:00010000:0.0:1783520935.883073:0:20097:0:(ofd_lvb.c:306:ofd_lvbo_update()) res: [0x2c0000401:0x8f7:0x0] updating lvb size from disk: 262144 -> 65536 00002000:00010000:0.0:1783520935.883078:0:20097:0:(ofd_lvb.c:313:ofd_lvbo_update()) res: [0x2c0000401:0x8f7:0x0] updating lvb mtime from disk: 1783520936 -> 1783520936 00002000:00010000:0.0:1783520935.883081:0:20097:0:(ofd_lvb.c:319:ofd_lvbo_update()) res: [0x2c0000401:0x8f7:0x0] updating lvb atime from disk: 0 -> 0 00002000:00010000:0.0:1783520935.883084:0:20097:0:(ofd_lvb.c:325:ofd_lvbo_update()) res: [0x2c0000401:0x8f7:0x0] updating lvb ctime from disk: 1783520936 -> 1783520936 00002000:00010000:0.0:1783520935.883086:0:20097:0:(ofd_lvb.c:331:ofd_lvbo_update()) res: [0x2c0000401:0x8f7:0x0] updating lvb blocks from disk: 256 -> 0 00000100:00100000:0.0:1783520935.883122:0:20097:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634571500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_io00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+667:2389:x1870153002408192:12345-192.168.204.33@tcp:10:dd.0 Request processed in 575us (826us total) trans 4294975903 rc 0/0 00000100:00100000:1.0:1783520935.883128:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607c60000 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.883135:0:20097:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 34736 00002000:00010000:0.0:1783520935.883143:0:20097:0:(ofd_dev.c:2825:ofd_punch_hpreq_check()) lustre-OST0001: refresh locks: 11811161089/2295 (65536->18446744073709551615) 00002000:00100000:0.0:1783520935.883147:0:20097:0:(ofd_dev.c:2565:ofd_prolong_extent_locks()) Prolong locks for req ffff943634571500 with x1870153002408192 ext(65536->18446744073709551615) 00002000:00010000:0.0:1783520935.883155:0:20097:0:(ofd_dev.c:2830:ofd_punch_hpreq_check()) lustre-OST0001: refreshed 0 locks timeout for req ffff943634571500. 00010000:00010000:1.0:1783520935.883233:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025a4d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x664:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.883245:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025a4d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x664:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.883257:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025a4d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x664:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.883266:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025a4d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x664:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.883275:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025a62 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x666:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.883285:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.883285:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025a62 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x666:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.883299:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025a62 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x666:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.883300:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.883308:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025a4d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x664:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.883312:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025a62 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x666:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.883324:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.883330:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025a4d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x664:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.883338:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.883339:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025a4d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x664:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.883387:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025a4d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x664:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.883396:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025a4d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x664:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.883429:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.883435:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025a62 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x666:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.883442:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.883445:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025a62 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x666:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.883454:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025a62 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x666:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.883466:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025a62 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x666:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.883474:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025a62 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x666:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.883489:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+338:3628:x1870152993135744:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3764us (4789us total) trans 4294975904 rc 0/0 00000100:00100000:0.0:1783520935.883514:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44404 00010000:00010000:1.0:1783520935.883520:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.883524:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.883529:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:2.0:1783520935.883594:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435161c0e00/2, rc 0, transno 4294975904 00000100:00100000:2.0:1783520935.883619:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993135744:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.883649:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.883656:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e98e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+338:3626:x1870152993135104:12345-0@lo:6:osp-syn-1-0.0 Request processed in 10408us (10503us total) trans 4294975905 rc 0/0 00000100:00100000:1.0:1783520935.883669:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44401 00000004:00080000:1.0:1783520935.883719:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e005c00/2, rc 0, transno 4294975905 00000100:00100000:0.0:1783520935.883724:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508059180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993137536:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.883732:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e005c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993135104:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.883762:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.883786:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993137536 00000100:00100000:1.0:1783520935.883791:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993137536 00000100:00100000:1.0:1783520935.883849:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44411 00000100:00100000:1.0:1783520935.883862:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+338:3627:x1870152993137536:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.883869:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993137664:0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.883877:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1645 count 1 00010000:00010000:1.0:1783520935.883895:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025a93 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x66d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.883911:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f19357bc/0xd5401cbabc0202b1 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x66d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bbc expref: 666 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.883912:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.883925:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x66d:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.883930:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x66d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.883935:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025a93 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x66d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.883965:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f19357bc/0xd5401cbabc0202b1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x66d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bbc expref: 666 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.883975:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993137664 00010000:00010000:1.0:1783520935.883980:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f19357bc/0xd5401cbabc0202b1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x66d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bbc expref: 666 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.883982:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993137664 00000100:00100000:1.0:1783520935.884003:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cbaad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993137792:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.884035:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44412 00000100:00100000:0.0:1783520935.884047:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e99f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+339:3628:x1870152993137664:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.884066:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1646 count 1 00000100:00100000:1.0:1783520935.884075:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f000 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.884083:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025a9a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x66e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.884099:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b8ab2f28/0xd5401cbabc022946 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x66e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3036 expref: 666 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.884107:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x66e:0x0].0x0 since=36s v=(0 0) 00010000:00010000:0.0:1783520935.884111:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x66e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.884115:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025a9a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x66e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.884145:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b8ab2f28/0xd5401cbabc022946 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x66e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3036 expref: 666 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.884156:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b8ab2f28/0xd5401cbabc022946 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x66e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3036 expref: 666 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.884167:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993137920:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.884203:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.885852:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350805b800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993136768:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.885887:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025a77 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x66a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.885941:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e98a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993136512:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.885953:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943634571180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993136640:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.885984:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025a70 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x669:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.885987:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025a69 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x667:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.886077:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350805b100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993136896:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.886099:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025a7e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x668:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.886106:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba8380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993137152:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.886140:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025a85 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x66b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.886185:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.886276:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.886307:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.886334:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.886351:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.886427:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.886526:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cbaad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993137792:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.886547:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.886551:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025a93 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x66d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.886590:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993137920:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.886617:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025a9a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x66e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.886618:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.886655:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002408448 00000100:00100000:1.0:1783520935.886663:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002408448 00010000:00010000:1.0:1783520935.886710:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ee012eaf/0xd5401cbabc020287 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x667:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b68 expref: 667 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.886738:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002408704 00000100:00100000:1.0:1783520935.886742:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002408704 00010000:00010000:1.0:1783520935.886757:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b76e260d/0xd5401cbabc02028e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x669:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b84 expref: 668 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.886769:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002408576 00000100:00100000:1.0:1783520935.886771:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002408576 00010000:00010000:1.0:1783520935.886784:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e4cb304f/0xd5401cbabc020295 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x66a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b92 expref: 669 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.886922:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002408832 00000100:00100000:1.0:1783520935.886925:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002408832 00010000:00010000:1.0:1783520935.886944:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a8381c9c/0xd5401cbabc020280 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x668:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b76 expref: 670 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.886961:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002408960 00000100:00100000:1.0:1783520935.886964:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002408960 00010000:00010000:1.0:1783520935.886985:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000dfec228a/0xd5401cbabc02029c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x66b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ba0 expref: 671 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.887003:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002409088 00000100:00100000:1.0:1783520935.887005:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002409088 00010000:00010000:1.0:1783520935.887029:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004f01fcd0/0xd5401cbabc0202a3 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x66c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bae expref: 672 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.887047:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002409216 00000100:00100000:1.0:1783520935.887051:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002409216 00010000:00010000:1.0:1783520935.887065:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f19357bc/0xd5401cbabc0202b1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x66d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bbc expref: 673 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.887083:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002409344 00000100:00100000:1.0:1783520935.887086:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002409344 00010000:00010000:1.0:1783520935.887099:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b8ab2f28/0xd5401cbabc022946 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x66e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3036 expref: 674 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.887112:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35997 00000100:00100000:1.0:1783520935.887122:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cba8380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+674:2388:x1870153002408448:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.887139:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.887146:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ee012eaf/0xd5401cbabc020287 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x667:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b68 expref: 674 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.887158:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ee012eaf/0xd5401cbabc020287 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x667:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b68 expref: 674 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.887187:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ee012eaf/0xd5401cbabc020287 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x667:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b68 expref: 674 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.887203:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x667:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.887207:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x667:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.887213:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025a69 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x667:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.887224:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025a69 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x667:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.887263:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.887298:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025a69 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x667:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.887311:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025a69 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x667:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.887324:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025a69 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x667:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.887334:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025a69 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x667:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.887351:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.887652:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+340:3626:x1870152993135872:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6744us (7246us total) trans 4294975906 rc 0/0 00000100:00100000:1.0:1783520935.887662:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44405 00000004:00080000:1.0:1783520935.887730:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e9bb80/2, rc 0, transno 4294975906 00000100:00100000:1.0:1783520935.887752:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993135872:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.887790:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35998 00000100:00100000:1.0:1783520935.887800:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cba9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+673:2388:x1870153002408704:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.887812:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.887819:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b76e260d/0xd5401cbabc02028e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x669:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b84 expref: 673 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.887837:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b76e260d/0xd5401cbabc02028e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x669:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b84 expref: 673 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.887855:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b76e260d/0xd5401cbabc02028e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x669:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b84 expref: 673 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.887875:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x669:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.887879:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x669:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.887883:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025a70 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x669:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.887893:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025a70 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x669:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.887908:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993138048:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.887921:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.887947:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025a70 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x669:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.887957:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025a70 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x669:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.887967:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025a70 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x669:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.887968:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.887980:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025a70 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x669:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.887998:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:0.0:1783520935.888051:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993138048 00000100:00100000:0.0:1783520935.888057:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993138048 00000100:00100000:0.0:1783520935.888106:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44413 00000100:00100000:1.0:1783520935.888132:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+341:3627:x1870152993136256:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7189us (7332us total) trans 4294975907 rc 0/0 00000100:00100000:1.0:1783520935.888142:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44406 00000004:00080000:0.0:1783520935.888149:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e9b480/2, rc 0, transno 4294975907 00000100:00100000:0.0:1783520935.888163:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993136256:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.888219:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cba9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+672:2388:x1870153002408704:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 421us (1941us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.888232:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35998 00000100:00100000:0.0:1783520935.888273:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1783520935.888274:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 35999 00000100:00100000:0.0:1783520935.888289:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e99880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993138176:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.888290:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cba9180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+671:2388:x1870153002408576:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.888310:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.888316:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e4cb304f/0xd5401cbabc020295 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x66a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b92 expref: 671 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.888332:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e4cb304f/0xd5401cbabc020295 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x66a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b92 expref: 671 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.888342:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.888353:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36000 00010000:00010000:1.0:1783520935.888355:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e4cb304f/0xd5401cbabc020295 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x66a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b92 expref: 671 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.888409:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x66a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.888414:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x66a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.888421:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025a77 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.888425:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94361b4a6450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+670:2388:x1870153002408832:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.888444:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993138176 00000100:00100000:0.0:1783520935.888452:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993138176 00010000:00010000:1.0:1783520935.888456:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025a77 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x66a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.888470:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.888485:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a8381c9c/0xd5401cbabc020280 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x668:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b76 expref: 670 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.888488:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.888525:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025a77 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.888529:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a8381c9c/0xd5401cbabc020280 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x668:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b76 expref: 670 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.888541:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cba9180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+670:2388:x1870153002408576:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 251us (2226us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.888562:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025a77 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.888581:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35999 00010000:00010000:0.0:1783520935.888606:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025a77 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.888616:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025a77 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x66a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.888633:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36001 00010000:00010000:0.0:1783520935.888643:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to regular blp list, 3 blwis in pool 00000100:00100000:1.0:1783520935.888649:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cbaa300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+669:2389:x1870153002408960:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.888662:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.888668:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000dfec228a/0xd5401cbabc02029c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x66b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ba0 expref: 669 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.888688:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000dfec228a/0xd5401cbabc02029c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x66b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ba0 expref: 669 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.888689:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a8381c9c/0xd5401cbabc020280 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x668:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0b76 expref: 669 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.888715:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000dfec228a/0xd5401cbabc02029c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x66b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ba0 expref: 669 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.888727:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x66b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.888732:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x66b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.888737:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025a85 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.888772:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025a85 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x66b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.888776:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5b8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+342:3627:x1870152993136384:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7628us (7865us total) trans 4294975908 rc 0/0 00010000:00010000:2.0:1783520935.888788:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x668:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.888796:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44407 00010000:00010000:2.0:1783520935.888797:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x668:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.888808:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025a7e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x668:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.888813:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.888825:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025a7e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x668:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.888836:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44414 00000100:00100000:1.0:1783520935.888848:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cbaa300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+667:2389:x1870153002408960:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 206us (2495us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.888856:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e98a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+341:3625:x1870152993138176:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.888864:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36001 00010000:00010000:2.0:1783520935.888884:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00002000:00080000:0.0:1783520935.888889:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1648 count 1 00000100:00100000:1.0:1783520935.888902:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36002 00000100:00100000:1.0:1783520935.888912:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cba9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+666:2389:x1870153002409088:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.888920:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025aa1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x670:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.888921:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.888929:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004f01fcd0/0xd5401cbabc0202a3 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x66c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bae expref: 666 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.888947:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006886eafb/0xd5401cbabc0207b2 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x670:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0be6 expref: 666 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.888956:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004f01fcd0/0xd5401cbabc0202a3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x66c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bae expref: 666 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.888966:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94361b4a6450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+666:2388:x1870153002408832:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 546us (2615us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.888972:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x670:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.888976:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x670:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.888980:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025aa1 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x670:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.888984:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x66c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.889002:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x66c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.889015:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025a8c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.889020:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36000 00010000:00010000:1.0:1783520935.889039:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025a8c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x66c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.889049:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006886eafb/0xd5401cbabc0207b2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x670:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0be6 expref: 666 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.889082:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.889082:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006886eafb/0xd5401cbabc0207b2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x670:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0be6 expref: 665 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.889092:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36003 00000100:00100000:0.0:1783520935.889120:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350805b100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993138304:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.889127:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cba9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+665:2389:x1870153002409088:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 211us (2696us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.889138:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cbab480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+665:2390:x1870153002409216:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.889147:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36002 00010000:00010000:2.0:1783520935.889160:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.889173:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f19357bc/0xd5401cbabc0202b1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x66d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bbc expref: 664 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.889185:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36004 00010000:00010000:2.0:1783520935.889200:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f19357bc/0xd5401cbabc0202b1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x66d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bbc expref: 664 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.889206:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cbab800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+664:2390:x1870153002409344:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.889220:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470e40 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.889226:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.889235:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b8ab2f28/0xd5401cbabc022946 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x66e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3036 expref: 664 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.889247:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:0.0:1783520935.889264:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025a69 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x667:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.889275:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b8ab2f28/0xd5401cbabc022946 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x66e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3036 expref: 664 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.889321:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.889324:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b8ab2f28/0xd5401cbabc022946 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x66e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3036 expref: 664 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.889328:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f19357bc/0xd5401cbabc0202b1 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x66d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bbc expref: 664 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.889330:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025a69 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x667:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.889344:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x66e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.889389:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x66e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.889389:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025a69 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x667:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.889398:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025a9a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.889406:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025a9a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x66e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.889412:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025a69 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x667:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.889438:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x66d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.889450:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025a69 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x667:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.889458:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x66d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.889478:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025a93 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.889488:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.889505:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025a93 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x66d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.889513:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520935.889521:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025a70 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x669:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.889529:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cbab800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+662:2390:x1870153002409344:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 333us (2913us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.889533:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520935.889538:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36004 00010000:00010000:0.0:1783520935.889542:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025a70 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x669:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.889545:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.889557:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025a70 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x669:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.889573:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025a85 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.889575:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025a70 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x669:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.889598:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cbab480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+661:2390:x1870153002409216:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 475us (3045us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.889600:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025a85 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.889602:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025a70 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x669:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.889635:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025a85 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.889652:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36003 00010000:00010000:1.0:1783520935.889654:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025a85 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x66b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.889674:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1783520935.889679:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.889680:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025a77 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x66a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.889691:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.889697:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025a77 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x66a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.889708:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.889708:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025a77 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x66a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.889728:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025a85 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x66b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.889735:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025a77 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x66a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.889751:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.889754:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025a77 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x66a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.889761:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025a85 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x66b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.889780:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025a85 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x66b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.889794:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.889798:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508059f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+342:3625:x1870152993137024:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7789us (7941us total) trans 4294975909 rc 0/0 00010000:00010000:0.0:1783520935.889802:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.889803:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025a85 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x66b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.889811:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44409 00010000:00010000:0.0:1783520935.889812:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.889828:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025a85 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x66b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.889866:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9dc00/2, rc 0, transno 4294975909 00000100:00100000:1.0:1783520935.889868:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cba8380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+660:2388:x1870153002408448:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 2743us (3677us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.889870:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.889876:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.889885:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.889886:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993137024:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.889887:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 35997 00010000:00010000:2.0:1783520935.889907:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.889932:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.889978:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1783520935.889996:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350805b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993138432:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.890014:0:12734:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004f01fcd0/0xd5401cbabc0202a3 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x66c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bae expref: 659 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.890035:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cbaa680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993137408:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.890050:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.890061:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025a8c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.890088:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.890089:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993138432 00000100:00100000:1.0:1783520935.890096:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993138432 00000004:00080000:0.0:1783520935.890144:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e9a300/2, rc 0, transno 4294975908 00000100:00100000:1.0:1783520935.890146:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44415 00000100:00100000:1.0:1783520935.890163:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d320700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+342:3628:x1870152993138432:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.890167:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993136384:0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.890179:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1649 count 1 00010000:00010000:1.0:1783520935.890197:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025aa8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x671:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.890211:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002e1cff96/0xd5401cbabc0207b9 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x671:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bf4 expref: 658 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.890217:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x671:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.890221:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x671:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.890226:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025aa8 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x671:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.890255:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d322d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993138560:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.890262:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002e1cff96/0xd5401cbabc0207b9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x671:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bf4 expref: 658 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.890278:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002e1cff96/0xd5401cbabc0207b9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x671:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bf4 expref: 658 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.890291:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cbaa680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993138688:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.890300:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.890391:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607c60000 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.890420:0:16376:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002409600 00000100:00100000:1.0:1783520935.890426:0:16376:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002409600 00010000:00010000:0.0:1783520935.890477:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025a9a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.890484:0:16376:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 9627 00010000:00010000:0.0:1783520935.890505:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025a9a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.890510:0:16376:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943608928c50 pname:cluuid+ref:pid:xid:nid:opc:job mdt_rdpg00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+30:251804:x1870153002409600:12345-192.168.204.33@tcp:35:dd.0 00010000:00010000:0.0:1783520935.890525:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025a9a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.890534:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025a9a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x66e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.890559:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.890574:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025a7e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x668:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.890585:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025a7e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x668:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.890598:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025a7e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x668:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.890607:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025a7e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x668:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.890632:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520935.890648:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520935.890655:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025a9a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x66e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.890663:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.890668:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025a9a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x66e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.890678:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025a9a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x66e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.890693:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e99f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+343:3628:x1870152993137664:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6647us (6780us total) trans 4294975910 rc 0/0 00010000:00010000:2.0:1783520935.890699:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025a9a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x66e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.890707:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44412 00010000:00010000:2.0:1783520935.890719:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025a9a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x66e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.890740:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993138560 00000100:00100000:0.0:1783520935.890746:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993138560 00010000:00010000:2.0:1783520935.890749:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.890753:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025a7e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x668:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.890758:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.890761:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025a7e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x668:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.890769:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025a7e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x668:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.890789:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025a7e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x668:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.890794:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025a7e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x668:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.890801:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44416 00000100:00100000:0.0:1783520935.890825:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d321f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+343:3628:x1870152993138560:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.890829:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.890834:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.890836:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00002000:00080000:0.0:1783520935.890842:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1650 count 1 00000100:00100000:1.0:1783520935.890859:0:16376:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943608928c50 pname:cluuid+ref:pid:xid:nid:opc:job mdt_rdpg00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+31:251804:x1870153002409600:12345-192.168.204.33@tcp:35:dd.0 Request processed in 361us (782us total) trans 8589957901 rc 0/0 00010000:00010000:0.0:1783520935.890865:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025aaf lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x672:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.890870:0:16376:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 9627 00010000:00010000:0.0:1783520935.890886:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000039dccd12/0xd5401cbabc0207c0 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x672:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c02 expref: 658 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.890894:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x672:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.890903:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x672:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.890909:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025aaf lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x672:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.890942:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000039dccd12/0xd5401cbabc0207c0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x672:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c02 expref: 658 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.890952:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000039dccd12/0xd5401cbabc0207c0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x672:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c02 expref: 658 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.890968:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d321180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993138816:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.891019:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470b40 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.891031:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025a8c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.891042:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025a8c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.891055:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634444450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+344:3625:x1870152993136128:12345-0@lo:6:osp-syn-1-0.0 Request processed in 9552us (10072us total) trans 4294975911 rc 0/0 00010000:00010000:1.0:1783520935.891055:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025a8c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.891058:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e99c00/2, rc 0, transno 4294975911 00000100:00100000:0.0:1783520935.891079:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e99c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993136128:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.891095:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44408 00010000:00010000:1.0:1783520935.891104:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025a8c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x66c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.891132:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.891157:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1783520935.891173:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993138944:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.891227:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.891291:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cbab100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+344:3626:x1870152993137280:12345-0@lo:6:osp-syn-1-0.0 Request processed in 8447us (8573us total) trans 4294975912 rc 0/0 00000100:00100000:1.0:1783520935.891305:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44410 00000004:00080000:0.0:1783520935.891306:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cba9500/2, rc 0, transno 4294975912 00000100:00100000:0.0:1783520935.891320:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993137280:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.891336:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993138944 00010000:00010000:2.0:1783520935.891337:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520935.891343:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993138944 00010000:00010000:2.0:1783520935.891350:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025a8c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x66c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.891416:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.891425:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025a8c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x66c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.891437:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025a8c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x66c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.891440:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.891445:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44417 00010000:00010000:2.0:1783520935.891452:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025a8c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x66c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.891459:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1783520935.891466:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d320000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+344:3626:x1870152993138944:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.891484:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993139072:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.891486:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025a8c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x66c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.891509:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1651 count 1 00010000:00010000:1.0:1783520935.891536:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025ab6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x673:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.891546:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.891551:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003584d90a/0xd5401cbabc0207c7 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x673:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c10 expref: 658 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.891554:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.891565:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.891566:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.891572:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x673:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.891577:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x673:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.891582:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025ab6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x673:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.891616:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993139072 00010000:00010000:1.0:1783520935.891617:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003584d90a/0xd5401cbabc0207c7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x673:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c10 expref: 658 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.891632:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993139072 00010000:00010000:1.0:1783520935.891640:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003584d90a/0xd5401cbabc0207c7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x673:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c10 expref: 658 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.891656:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435161c1f80/2, rc 0, transno 4294975910 00000100:00100000:1.0:1783520935.891659:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba9880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993139200:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.891675:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993137664:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.891691:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44418 00000100:00100000:2.0:1783520935.891714:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d321880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+345:3625:x1870152993139072:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.891757:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e840 going to sleep for 11 seconds 00002000:00080000:2.0:1783520935.891758:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1652 count 1 00000100:00100000:0.0:1783520935.891764:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1783520935.891780:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993139328:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.891793:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025abd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x674:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.891813:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000588b93b9/0xd5401cbabc0207ce lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x674:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c1e expref: 658 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.891826:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.891841:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x674:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.891847:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x674:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.891853:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025abd lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x674:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.891885:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993139328 00010000:00010000:2.0:1783520935.891892:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000588b93b9/0xd5401cbabc0207ce lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x674:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c1e expref: 658 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.891894:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993139328 00010000:00010000:2.0:1783520935.891923:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000588b93b9/0xd5401cbabc0207ce lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x674:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c1e expref: 658 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.891941:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943634571c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993139456:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.891971:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44419 00000100:00100000:1.0:1783520935.891991:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d322300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+346:3628:x1870152993139328:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.891999:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676d80 going to sleep for 11 seconds 00002000:00080000:1.0:1783520935.892010:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1653 count 1 00010000:00010000:1.0:1783520935.892027:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025ac4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x675:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.892038:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e419cc2d/0xd5401cbabc0207d5 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x675:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c2c expref: 658 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.892050:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+346:3625:x1870152993138048:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.892050:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x675:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.892059:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x675:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.892071:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025ac4 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x675:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.892085:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1647 count 1 00010000:00010000:1.0:1783520935.892096:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e419cc2d/0xd5401cbabc0207d5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x675:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c2c expref: 658 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.892102:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025acb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x66f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.892109:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e419cc2d/0xd5401cbabc0207d5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x675:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c2c expref: 658 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.892128:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001864bc04/0xd5401cbabc0202b8 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x66f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bd8 expref: 658 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.892143:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x66f:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.892145:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x66f:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.892151:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba9180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993139584:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.892152:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025acb lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x66f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.892197:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001864bc04/0xd5401cbabc0202b8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x66f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bd8 expref: 658 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.892213:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001864bc04/0xd5401cbabc0202b8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x66f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bd8 expref: 658 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.892216:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e600 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.892235:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943634571f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993139712:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.892266:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676840 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.892282:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025a93 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.892291:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025a93 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.892301:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025a93 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.892308:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025a93 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x66d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.892327:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.892339:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.892345:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025a93 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x66d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.892350:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.892353:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025a93 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x66d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.892394:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025a93 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x66d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.892417:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025a93 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x66d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.892429:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350805b100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993138304:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.892430:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025a93 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x66d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.892477:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.892479:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025aa1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x670:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.892480:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.892484:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.892520:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cbaa680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993138688:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.892540:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025aa8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x671:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.892618:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.892646:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+347:3627:x1870152993137536:12345-0@lo:6:osp-syn-1-0.0 Request processed in 8785us (8881us total) trans 4294975913 rc 0/0 00000100:00100000:2.0:1783520935.892656:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44411 00000004:00080000:0.0:1783520935.892667:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943508059180/2, rc 0, transno 4294975913 00000100:00100000:0.0:1783520935.892680:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508059180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993137536:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.892730:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002409728 00000100:00100000:1.0:1783520935.892736:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002409728 00000100:00100000:0.0:1783520935.892761:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d323480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993139840:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.892772:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006886eafb/0xd5401cbabc0207b2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x670:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0be6 expref: 659 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.892817:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.892835:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36005 00000100:00100000:1.0:1783520935.892848:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cbaa680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+659:2388:x1870153002409728:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.892861:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.892869:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006886eafb/0xd5401cbabc0207b2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x670:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0be6 expref: 659 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.892893:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006886eafb/0xd5401cbabc0207b2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x670:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0be6 expref: 659 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.892897:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993139840 00000100:00100000:0.0:1783520935.892905:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993139840 00010000:00010000:1.0:1783520935.892923:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006886eafb/0xd5401cbabc0207b2 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x670:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0be6 expref: 659 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.892943:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x670:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.892947:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x670:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.892954:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025aa1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x670:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.892955:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44420 00000100:00100000:0.0:1783520935.892965:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d321c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+347:3628:x1870152993139840:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.892976:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025aa1 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x670:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.892981:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1654 count 1 00010000:00010000:0.0:1783520935.892999:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025ad2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x676:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.893006:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.893011:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a0ec3890/0xd5401cbabc022ba0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x676:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3290 expref: 658 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.893017:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x676:0x0].0x0 since=36s v=(0 0) 00010000:00010000:0.0:1783520935.893020:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x676:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.893023:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025ad2 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x676:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.893038:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a0ec3890/0xd5401cbabc022ba0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x676:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3290 expref: 658 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.893048:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cbaa680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+658:2388:x1870153002409728:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 201us (429us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.893049:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a0ec3890/0xd5401cbabc022ba0 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x676:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3290 expref: 658 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.893065:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36005 00000100:00100000:0.0:1783520935.893067:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d320a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993139968:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.893153:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470e40 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.893163:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025aa1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x670:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.893175:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025aa1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x670:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.893186:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025aa1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x670:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.893195:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025aa1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x670:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.893224:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.893232:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.893240:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025aa1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x670:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.893251:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.893255:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025aa1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x670:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.893265:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025aa1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x670:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.893283:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025aa1 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x670:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.893291:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025aa1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x670:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.893321:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.893329:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.893332:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.893351:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e98a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+348:3625:x1870152993138176:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4500us (5014us total) trans 4294975914 rc 0/0 00000100:00100000:1.0:1783520935.893488:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44414 00000004:00080000:0.0:1783520935.893576:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e99880/2, rc 0, transno 4294975914 00000100:00100000:0.0:1783520935.893594:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e99880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993138176:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.893635:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.893683:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1783520935.893686:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002409856 00000100:00100000:2.0:1783520935.893697:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002409856 00000100:00100000:0.0:1783520935.893702:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993140096:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.893742:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002e1cff96/0xd5401cbabc0207b9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x671:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bf4 expref: 658 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.893753:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.893812:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36006 00000100:00100000:1.0:1783520935.893827:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba9880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993139200:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.893829:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cba8380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+658:2390:x1870153002409856:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.893853:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.893857:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d321180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993138816:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.893865:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002e1cff96/0xd5401cbabc0207b9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x671:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bf4 expref: 658 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.893866:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025ab6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x673:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.893882:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025aaf lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x672:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.893938:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002e1cff96/0xd5401cbabc0207b9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x671:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bf4 expref: 658 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.893947:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993140096 00000100:00100000:0.0:1783520935.893953:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993140096 00010000:00010000:2.0:1783520935.893958:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002e1cff96/0xd5401cbabc0207b9 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x671:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bf4 expref: 658 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.893973:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x671:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.893976:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x671:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.893980:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025aa8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x671:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.893991:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025aa8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x671:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.893997:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44421 00000100:00100000:0.0:1783520935.894017:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351635a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+348:3626:x1870152993140096:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.894034:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1655 count 1 00010000:00010000:2.0:1783520935.894039:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.894053:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025ad9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x677:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.894069:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005d23ea85/0xd5401cbabc0207e3 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x677:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c48 expref: 657 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.894076:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x677:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.894079:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x677:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.894084:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025aa8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x671:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.894085:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025ad9 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x677:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.894110:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025aa8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x671:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.894112:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cba8380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+657:2390:x1870153002409856:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 286us (478us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.894129:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005d23ea85/0xd5401cbabc0207e3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x677:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c48 expref: 657 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.894148:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36006 00010000:00010000:1.0:1783520935.894171:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025aa8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x671:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.894183:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005d23ea85/0xd5401cbabc0207e3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x677:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c48 expref: 656 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.894198:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.894214:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025aa8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x671:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.894229:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351635b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993140224:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.894252:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002409984 00010000:00010000:1.0:1783520935.894256:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520935.894261:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002409984 00010000:00010000:0.0:1783520935.894273:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.894283:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025aa8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x671:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.894296:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbaa00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.894297:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000039dccd12/0xd5401cbabc0207c0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x672:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c02 expref: 657 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.894306:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025aa8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x671:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.894327:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025aa8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x671:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.894342:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025aa8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x671:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.894349:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d320700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+349:3628:x1870152993138432:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4192us (4299us total) trans 4294975915 rc 0/0 00010000:00010000:0.0:1783520935.894354:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025aa8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x671:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.894385:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44415 00010000:00010000:0.0:1783520935.894428:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.894434:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.894445:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.894481:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470b40 going to sleep for 11 seconds 00000004:00080000:0.0:1783520935.894489:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350805b800/2, rc 0, transno 4294975915 00000100:00100000:0.0:1783520935.894517:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350805b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993138432:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.894545:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36007 00000100:00100000:3.0:1783520935.894561:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94361b4a5c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+657:2388:x1870153002409984:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.894576:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.894583:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000039dccd12/0xd5401cbabc0207c0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x672:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c02 expref: 657 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.894585:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351635ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993140352:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.894618:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000039dccd12/0xd5401cbabc0207c0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x672:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c02 expref: 657 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.894634:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.894643:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000039dccd12/0xd5401cbabc0207c0 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x672:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c02 expref: 657 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.894662:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x672:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.894666:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x672:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.894672:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025aaf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x672:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.894683:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025aaf lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x672:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.894690:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993140352 00000100:00100000:2.0:1783520935.894695:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993140352 00010000:00010000:3.0:1783520935.894710:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.894734:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44422 00010000:00010000:0.0:1783520935.894740:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025aaf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x672:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.894750:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516358a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+349:3626:x1870152993140352:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.894770:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94361b4a5c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+656:2388:x1870153002409984:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 207us (589us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.894780:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025aaf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x672:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.894781:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1656 count 1 00000100:00100000:3.0:1783520935.894797:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36007 00010000:00010000:2.0:1783520935.894805:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025ae0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x678:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.894811:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025aaf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x672:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.894819:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008b8fd7fa/0xd5401cbabc0207ea lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x678:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c56 expref: 655 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.894850:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025aaf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x672:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.894851:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x678:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.894871:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x678:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.894884:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025ae0 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x678:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.894907:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.894921:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.894924:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008b8fd7fa/0xd5401cbabc0207ea lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x678:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c56 expref: 655 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.894947:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008b8fd7fa/0xd5401cbabc0207ea lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x678:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c56 expref: 655 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.894961:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943634573b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993140480:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.894985:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520935.894985:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002410112 00000100:00100000:3.0:1783520935.894991:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba9180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993139584:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.894991:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002410112 00010000:00010000:2.0:1783520935.894993:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025aaf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x672:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.895005:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.895012:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025aaf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x672:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.895026:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003584d90a/0xd5401cbabc0207c7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x673:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c10 expref: 656 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.895030:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025aaf lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x672:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.895032:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025ac4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x675:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.895046:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d321f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+350:3628:x1870152993138560:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4222us (4738us total) trans 4294975916 rc 0/0 00000100:00100000:1.0:1783520935.895067:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36008 00010000:00010000:2.0:1783520935.895078:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025aaf lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x672:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.895091:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e471f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+656:2389:x1870153002410112:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.895093:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44416 00010000:00010000:2.0:1783520935.895098:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025aaf lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x672:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.895103:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.895109:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003584d90a/0xd5401cbabc0207c7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x673:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c10 expref: 656 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.895124:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003584d90a/0xd5401cbabc0207c7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x673:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c10 expref: 656 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.895127:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.895132:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.895141:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.895145:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003584d90a/0xd5401cbabc0207c7 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x673:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c10 expref: 656 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.895159:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x673:0x0].0x0 since=0s v=(0 0) 00000004:00080000:0.0:1783520935.895162:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363d322d80/2, rc 0, transno 4294975916 00010000:00010000:1.0:1783520935.895163:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x673:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.895174:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025ab6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x673:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.895180:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d322d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993138560:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.895199:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025ab6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x673:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.895208:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676a80 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.895221:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.895249:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e471f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+655:2389:x1870153002410112:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 165us (343us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.895253:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516359500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993140608:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.895270:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36008 00000100:00100000:0.0:1783520935.895308:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.895311:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025ab6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x673:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.895324:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025ab6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x673:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.895339:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025ab6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x673:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.895348:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025ab6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x673:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.895356:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993140608 00000100:00100000:0.0:1783520935.895391:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993140608 00010000:00010000:1.0:1783520935.895430:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.895437:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44423 00010000:00010000:2.0:1783520935.895441:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520935.895451:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516359880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+350:3627:x1870152993140608:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.895453:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025ab6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x673:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.895472:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb800 already unused, calling callback (ffffffffc0bf9e90) 00002000:00080000:0.0:1783520935.895472:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1657 count 1 00010000:00010000:2.0:1783520935.895487:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025ab6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x673:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.895509:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025ab6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x673:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.895509:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025ae7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x679:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.895559:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e8648021/0xd5401cbabc0207f1 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x679:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c64 expref: 654 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.895560:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025ab6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x673:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.895603:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x679:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.895608:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025ab6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x673:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.895611:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x679:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.895615:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d320000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+351:3626:x1870152993138944:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4144us (4375us total) trans 4294975917 rc 0/0 00010000:00010000:0.0:1783520935.895635:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025ae7 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x679:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.895645:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44417 00010000:00010000:2.0:1783520935.895652:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.895654:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.895658:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.895669:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e8648021/0xd5401cbabc0207f1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x679:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c64 expref: 654 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.895683:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e8648021/0xd5401cbabc0207f1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x679:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c64 expref: 654 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.895698:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351635aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993140736:192.168.204.33@tcp:104: 00000004:00080000:1.0:1783520935.895742:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363d322a00/2, rc 0, transno 4294975917 00000100:00100000:1.0:1783520935.895763:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993138944:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.895807:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470900 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.895907:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1783520935.895924:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993140864:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.895969:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943634571c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993139456:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.895969:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943634571f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993139712:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.895985:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.895998:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025abd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x674:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.895998:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025acb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x66f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.896038:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993140864 00000100:00100000:0.0:1783520935.896045:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993140864 00000100:00100000:2.0:1783520935.896103:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.896120:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d320a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993139968:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.896157:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025ad2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x676:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.896158:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.896158:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002410496 00000100:00100000:1.0:1783520935.896172:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002410496 00000100:00100000:0.0:1783520935.896200:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44424 00010000:00010000:1.0:1783520935.896208:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000588b93b9/0xd5401cbabc0207ce lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x674:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c1e expref: 655 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.896210:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516359c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+351:3627:x1870152993140864:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.896236:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1658 count 1 00010000:00010000:0.0:1783520935.896254:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025aee lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x67a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.896272:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007b27a38d/0xd5401cbabc0207f8 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x67a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c72 expref: 655 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.896276:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002410624 00000100:00100000:1.0:1783520935.896286:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002410624 00000100:00100000:2.0:1783520935.896289:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36009 00010000:00010000:0.0:1783520935.896297:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x67a:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.896304:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x67a:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.896306:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634571c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+656:2389:x1870153002410496:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.896313:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a0ec3890/0xd5401cbabc022ba0 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x676:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3290 expref: 656 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.896315:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025aee lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x67a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.896323:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.896338:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000588b93b9/0xd5401cbabc0207ce lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x674:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c1e expref: 656 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.896404:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000588b93b9/0xd5401cbabc0207ce lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x674:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c1e expref: 656 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.896412:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007b27a38d/0xd5401cbabc0207f8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x67a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c72 expref: 656 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.896432:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36010 00000100:00100000:1.0:1783520935.896458:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634570700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+656:2387:x1870153002410624:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.896460:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007b27a38d/0xd5401cbabc0207f8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x67a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c72 expref: 656 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.896465:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000588b93b9/0xd5401cbabc0207ce lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x674:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c1e expref: 656 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.896478:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.896490:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a0ec3890/0xd5401cbabc022ba0 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x676:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3290 expref: 656 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.896524:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516358700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993140992:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.896526:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x674:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.896527:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a0ec3890/0xd5401cbabc022ba0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x676:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3290 expref: 655 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.896540:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x674:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.896544:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025abd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x674:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.896550:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a0ec3890/0xd5401cbabc022ba0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x676:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3290 expref: 655 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.896554:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025abd lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x674:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.896564:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x676:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.896567:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x676:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.896571:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025ad2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x676:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.896579:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.896579:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025ad2 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x676:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.896602:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470e40 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.896611:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.896617:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634571c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+654:2389:x1870153002410496:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 316us (521us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.896625:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025ad2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x676:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.896627:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36009 00000100:00100000:1.0:1783520935.896640:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634570700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+654:2387:x1870153002410624:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 187us (482us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.896658:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025ad2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x676:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.896672:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36010 00010000:00010000:2.0:1783520935.896680:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025abd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x674:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.896726:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025ad2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x676:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.896734:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025abd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x674:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.896750:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025ad2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x676:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.896751:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.896758:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025abd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x674:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.896800:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025abd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x674:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.896805:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002410240 00000100:00100000:3.0:1783520935.896813:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002410240 00000100:00100000:1.0:1783520935.896814:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.896818:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.896826:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520935.896839:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e419cc2d/0xd5401cbabc0207d5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x675:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c2c expref: 653 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.896839:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002410368 00010000:00010000:2.0:1783520935.896840:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00000100:00100000:1.0:1783520935.896844:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002410368 00010000:00010000:2.0:1783520935.896848:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025ad2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x676:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.896860:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbebe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.896863:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001864bc04/0xd5401cbabc0202b8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x66f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bd8 expref: 654 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.896865:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025ad2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x676:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.896875:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025ad2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x676:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.896882:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36011 00000100:00100000:1.0:1783520935.896891:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36012 00010000:00010000:2.0:1783520935.896895:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025ad2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x676:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.896896:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cba9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+654:2389:x1870153002410240:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.896906:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cbaaa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+654:2387:x1870153002410368:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.896909:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025ad2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x676:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.896917:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.896921:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.896928:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e419cc2d/0xd5401cbabc0207d5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x675:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c2c expref: 654 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.896930:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.896930:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001864bc04/0xd5401cbabc0202b8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x66f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bd8 expref: 654 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.896936:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025abd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x674:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.896955:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.896955:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001864bc04/0xd5401cbabc0202b8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x66f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bd8 expref: 654 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.896960:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025abd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x674:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.896964:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e419cc2d/0xd5401cbabc0207d5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x675:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c2c expref: 654 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.896972:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025abd lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x674:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.896974:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001864bc04/0xd5401cbabc0202b8 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x66f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0bd8 expref: 654 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.896989:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d321c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+352:3628:x1870152993139840:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4017us (4168us total) trans 4294975918 rc 0/0 00010000:00010000:1.0:1783520935.896997:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x66f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.896999:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025abd lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x674:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.897002:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e419cc2d/0xd5401cbabc0207d5 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x675:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c2c expref: 653 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.897008:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x66f:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.897008:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44420 00010000:00010000:1.0:1783520935.897021:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025acb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.897028:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025abd lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x674:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.897039:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025acb lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x66f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.897042:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x675:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.897047:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.897053:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x675:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.897054:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.897060:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.897064:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025ac4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x675:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.897065:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.897074:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025ac4 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x675:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.897077:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363d323480/2, rc 0, transno 4294975918 00010000:00010000:1.0:1783520935.897079:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.897105:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.897105:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d323480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993139840:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.897122:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cbaaa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+652:2387:x1870153002410368:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 211us (298us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.897135:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36012 00000100:00100000:3.0:1783520935.897140:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cba9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+652:2389:x1870153002410240:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 247us (393us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.897148:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36011 00010000:00010000:1.0:1783520935.897180:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025acb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.897184:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025ac4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x675:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.897204:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025ac4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x675:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.897206:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025acb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.897209:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351635b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993141120:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.897242:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025ac4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x675:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.897243:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025acb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x66f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.897276:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025ac4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x675:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.897276:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.897278:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025acb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x66f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.897298:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d321880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+352:3625:x1870152993139072:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5593us (5743us total) trans 4294975919 rc 0/0 00000100:00100000:2.0:1783520935.897315:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44418 00010000:00010000:1.0:1783520935.897324:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520935.897332:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000004:00080000:0.0:1783520935.897332:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363d320e00/2, rc 0, transno 4294975919 00010000:00010000:2.0:1783520935.897347:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00000100:00100000:0.0:1783520935.897349:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993139072:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.897481:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+352:3625:x1870152993138048:12345-0@lo:6:osp-syn-1-0.0 Request processed in 9364us (9510us total) trans 4294975920 rc 0/0 00000004:00080000:0.0:1783520935.897491:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e9aa00/2, rc 0, transno 4294975920 00000100:00100000:1.0:1783520935.897492:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44413 00000100:00100000:0.0:1783520935.897513:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993138048:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.897528:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025acb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x66f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.897537:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993141120 00010000:00010000:2.0:1783520935.897538:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184a00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520935.897541:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993141120 00010000:00010000:2.0:1783520935.897542:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025acb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x66f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.897549:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025acb lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x66f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.897568:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025acb lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x66f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.897577:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025acb lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x66f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.897591:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44425 00010000:00010000:2.0:1783520935.897598:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.897603:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025ac4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x675:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.897608:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516359180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+353:3625:x1870152993141120:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.897610:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb400 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:3.0:1783520935.897617:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d322300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+353:3628:x1870152993139328:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5632us (5794us total) trans 4294975921 rc 0/0 00010000:00010000:2.0:1783520935.897618:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025ac4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x675:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.897628:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351635a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993141248:0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.897634:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1659 count 1 00000100:00100000:3.0:1783520935.897643:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44419 00010000:00010000:2.0:1783520935.897652:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025ac4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x675:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.897662:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025af5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x67b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.897664:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025ac4 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x675:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.897674:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025ac4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x675:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.897677:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008e368336/0xd5401cbabc0207ff lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x67b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c80 expref: 650 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.897692:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x67b:0x0].0x0 since=37s v=(0 0) 00000100:00100000:0.0:1783520935.897694:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.897697:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.897700:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x67b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.897707:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.897707:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025af5 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x67b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.897710:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.897736:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008e368336/0xd5401cbabc0207ff lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x67b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c80 expref: 650 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.897739:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.897751:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008e368336/0xd5401cbabc0207ff lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x67b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c80 expref: 650 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.897752:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993141248 00000100:00100000:2.0:1783520935.897756:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993141248 00000100:00100000:1.0:1783520935.897767:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cbabb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993141504:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.897783:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351635b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993141376:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.897793:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44426 00000100:00100000:2.0:1783520935.897805:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516358000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+353:3626:x1870152993141248:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.897818:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1660 count 1 00000100:00100000:1.0:1783520935.897824:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3eb40 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.897831:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.897833:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025afc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x67c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.897848:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b72c0a82/0xd5401cbabc020806 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x67c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c8e expref: 650 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.897851:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993141376 00000100:00100000:1.0:1783520935.897855:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993141376 00010000:00010000:2.0:1783520935.897856:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x67c:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.897860:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x67c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.897864:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025afc lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x67c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.897889:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b72c0a82/0xd5401cbabc020806 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x67c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c8e expref: 650 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.897904:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363d320380/2, rc 0, transno 4294975921 00010000:00010000:2.0:1783520935.897906:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b72c0a82/0xd5401cbabc020806 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x67c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c8e expref: 650 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.897906:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44427 00000100:00100000:1.0:1783520935.897922:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516359f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+354:3627:x1870152993141376:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.897928:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993139328:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.897929:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943634571f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993141632:192.168.204.33@tcp:104: 00002000:00080000:1.0:1783520935.897938:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1661 count 1 00010000:00010000:1.0:1783520935.897957:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025b03 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x67d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.897976:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676d80 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.897980:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000078aff5f/0xd5401cbabc02080d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x67d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c9c expref: 650 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.897990:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x67d:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.897993:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x67d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.897997:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025b03 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x67d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.898010:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1783520935.898023:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d320a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993141760:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.898027:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000078aff5f/0xd5401cbabc02080d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x67d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c9c expref: 650 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.898041:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000078aff5f/0xd5401cbabc02080d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x67d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c9c expref: 650 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.898054:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993141888:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.898075:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.898088:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.898137:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993141760 00000100:00100000:3.0:1783520935.898143:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993141760 00000100:00100000:2.0:1783520935.898146:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.898146:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351635b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993140224:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.898178:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025ad9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x677:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.898184:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44428 00000100:00100000:0.0:1783520935.898184:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351635aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993140736:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.898207:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d321180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+355:3627:x1870152993141760:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.898223:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002410752 00002000:00080000:3.0:1783520935.898224:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1662 count 1 00010000:00010000:0.0:1783520935.898224:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025ae7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x679:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.898231:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002410752 00010000:00010000:3.0:1783520935.898248:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025b0a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x67e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.898257:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005d23ea85/0xd5401cbabc0207e3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x677:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c48 expref: 651 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.898260:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.898274:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005fb589d4/0xd5401cbabc022a8f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x67e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f317f expref: 651 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.898283:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x67e:0x0].0x0 since=36s v=(0 0) 00010000:00010000:3.0:1783520935.898287:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x67e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520935.898292:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025b0a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x67e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.898293:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002410880 00000100:00100000:1.0:1783520935.898299:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002410880 00010000:00010000:1.0:1783520935.898315:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008b8fd7fa/0xd5401cbabc0207ea lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x678:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c56 expref: 652 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.898325:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005fb589d4/0xd5401cbabc022a8f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x67e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f317f expref: 652 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.898340:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005fb589d4/0xd5401cbabc022a8f lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x67e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f317f expref: 652 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.898340:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.898340:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36013 00000100:00100000:1.0:1783520935.898353:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634573100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+652:2387:x1870153002410752:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520935.898382:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e473100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993142016:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.898388:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002411008 00000100:00100000:2.0:1783520935.898393:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002411008 00010000:00010000:2.0:1783520935.898411:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e8648021/0xd5401cbabc0207f1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x679:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c64 expref: 653 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.898435:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435129679c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.898441:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36014 00000100:00100000:2.0:1783520935.898454:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94361b4a6c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+653:2390:x1870153002410880:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.898459:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36015 00010000:00010000:2.0:1783520935.898462:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.898468:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008b8fd7fa/0xd5401cbabc0207ea lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x678:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c56 expref: 653 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.898468:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634571180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+653:2390:x1870153002411008:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.898476:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.898479:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008b8fd7fa/0xd5401cbabc0207ea lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x678:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c56 expref: 653 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.898480:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e8648021/0xd5401cbabc0207f1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x679:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c64 expref: 653 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.898489:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e8648021/0xd5401cbabc0207f1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x679:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c64 expref: 653 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.898507:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x678:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.898510:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x678:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.898515:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025ae0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x678:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.898515:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e8648021/0xd5401cbabc0207f1 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x679:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c64 expref: 653 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.898525:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025ae0 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x678:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.898528:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x679:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.898531:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x679:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.898535:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025ae7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x679:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.898542:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.898548:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025ae7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x679:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.898564:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94361b4a6c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+652:2390:x1870153002410880:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 114us (373us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.898573:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36014 00010000:00010000:1.0:1783520935.898576:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.898601:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634571180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+651:2390:x1870153002411008:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 135us (263us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.898608:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36015 00010000:00010000:2.0:1783520935.898637:0:35276:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008b8fd7fa/0xd5401cbabc0207ea lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x678:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c56 expref: 650 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.898650:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943634573b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993140480:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.898650:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025ae7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x679:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.898660:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025ae7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x679:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.898669:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025ae7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x679:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.898674:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025ae0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x678:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.898678:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025ae7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x679:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.898682:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025ae0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x678:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.898689:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025ae0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x678:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.898697:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025ae0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x678:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.898701:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.898710:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025ae0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x678:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.898711:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.898723:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025ae7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x679:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.898730:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.898746:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.898747:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.898755:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025ae7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x679:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.898757:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025ae0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x678:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.898788:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516359880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+356:3627:x1870152993140608:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3338us (3481us total) trans 4294975922 rc 0/0 00010000:00010000:2.0:1783520935.898796:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.898797:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025ae7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x679:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.898798:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44423 00010000:00010000:2.0:1783520935.898806:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025ae0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x678:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.898819:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.898828:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005d23ea85/0xd5401cbabc0207e3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x677:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c48 expref: 649 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.898838:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025ae0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x678:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.898845:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005d23ea85/0xd5401cbabc0207e3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x677:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c48 expref: 649 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.898849:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025ae7 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x679:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.898854:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025ae0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x678:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.898858:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005d23ea85/0xd5401cbabc0207e3 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x677:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c48 expref: 649 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.898882:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025ae7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x679:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.898884:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025ae0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x678:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.898885:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x677:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.898897:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x677:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.898910:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025ad9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x677:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.898948:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.898948:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025ad9 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x677:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.898951:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.898953:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.898977:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.898978:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.898983:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.898987:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.899005:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634573100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+648:2387:x1870153002410752:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 652us (861us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.899017:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36013 00000004:00080000:0.0:1783520935.899029:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516359500/2, rc 0, transno 4294975922 00000100:00100000:0.0:1783520935.899047:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516359500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993140608:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.899048:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516358a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+356:3626:x1870152993140352:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4303us (4413us total) trans 4294975923 rc 0/0 00000100:00100000:2.0:1783520935.899070:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44422 00010000:00010000:1.0:1783520935.899074:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025ad9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x677:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.899082:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025ad9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x677:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.899090:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025ad9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x677:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.899099:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025ad9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x677:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.899116:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.899129:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.899136:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025ad9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x677:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.899144:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351635ad80/2, rc 0, transno 4294975923 00010000:00010000:2.0:1783520935.899145:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.899154:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025ad9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x677:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.899164:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351635ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993140352:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.899165:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025ad9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x677:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.899183:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:2.0:1783520935.899189:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025ad9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x677:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.899197:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351635a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+356:3626:x1870152993140096:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5181us (5451us total) trans 4294975924 rc 0/0 00000100:00100000:0.0:1783520935.899199:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a80700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993142144:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.899217:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44421 00000100:00100000:0.0:1783520935.899251:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.899267:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025ad9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x677:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.899294:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.899303:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.899305:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.899310:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516358700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993140992:192.168.204.33@tcp:104: 00000004:00080000:0.0:1783520935.899315:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363d323800/2, rc 0, transno 4294975924 00000100:00100000:3.0:1783520935.899325:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.899329:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993140096:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.899332:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025aee lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x67a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.899356:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002411136 00000100:00100000:3.0:1783520935.899392:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002411136 00010000:00010000:2.0:1783520935.899457:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007b27a38d/0xd5401cbabc0207f8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x67a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c72 expref: 648 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.899461:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1783520935.899478:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a82a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993142272:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.899492:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36016 00000100:00100000:2.0:1783520935.899514:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e473800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+648:2387:x1870153002411136:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.899526:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.899531:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993142144 00000100:00100000:0.0:1783520935.899536:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.899538:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007b27a38d/0xd5401cbabc0207f8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x67a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c72 expref: 648 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.899538:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993142144 00010000:00010000:2.0:1783520935.899555:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007b27a38d/0xd5401cbabc0207f8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x67a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c72 expref: 648 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.899574:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007b27a38d/0xd5401cbabc0207f8 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x67a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c72 expref: 648 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.899575:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993142272 00000100:00100000:1.0:1783520935.899578:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993142272 00000100:00100000:0.0:1783520935.899583:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:2.0:1783520935.899587:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x67a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.899593:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x67a:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.899597:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a82d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993142400:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.899600:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025aee lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x67a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.899606:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44429 00010000:00010000:2.0:1783520935.899617:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025aee lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x67a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.899620:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a81880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+357:3626:x1870152993142144:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.899635:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1663 count 1 00010000:00010000:2.0:1783520935.899642:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.899648:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.899652:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025b11 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x67f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.899669:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000906d822b/0xd5401cbabc02081b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x67f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cb8 expref: 647 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.899677:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x67f:0x0].0x0 since=37s v=(0 0) 00000100:00100000:2.0:1783520935.899679:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e473800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+647:2387:x1870153002411136:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 167us (357us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.899681:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x67f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.899685:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025b11 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x67f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.899686:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36016 00010000:00010000:1.0:1783520935.899711:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000906d822b/0xd5401cbabc02081b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x67f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cb8 expref: 646 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.899712:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993142400 00000100:00100000:0.0:1783520935.899722:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993142400 00000100:00100000:2.0:1783520935.899728:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44430 00010000:00010000:1.0:1783520935.899728:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000906d822b/0xd5401cbabc02081b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x67f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cb8 expref: 646 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.899739:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a83b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+358:3625:x1870152993142272:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.899740:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cbaaa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993142528:192.168.204.33@tcp:104: 00002000:00080000:2.0:1783520935.899752:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1664 count 1 00010000:00010000:2.0:1783520935.899767:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025b18 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x680:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.899781:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001daa4832/0xd5401cbabc020822 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x680:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cc6 expref: 646 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.899789:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44431 00000100:00100000:1.0:1783520935.899791:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e840 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.899800:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x680:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.899803:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x680:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.899808:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025b18 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x680:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.899811:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a81500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+358:3628:x1870152993142400:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.899816:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025aee lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x67a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.899833:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025aee lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x67a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.899841:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025aee lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x67a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.899848:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1665 count 1 00010000:00010000:2.0:1783520935.899851:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001daa4832/0xd5401cbabc020822 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x680:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cc6 expref: 646 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.899851:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025aee lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x67a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.899866:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001daa4832/0xd5401cbabc020822 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x680:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cc6 expref: 646 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.899874:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025b1f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x681:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.899877:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520935.899882:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943634573b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993142656:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.899907:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.899915:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025aee lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x67a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.899917:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e8b497e6/0xd5401cbabc020829 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x681:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cd4 expref: 646 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.899962:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x681:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.899965:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.899970:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x681:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.899975:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025aee lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x67a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.899978:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025b1f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x681:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.899980:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516359c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+359:3627:x1870152993140864:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3770us (4000us total) trans 4294975925 rc 0/0 00000100:00100000:1.0:1783520935.900015:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44424 00010000:00010000:2.0:1783520935.900036:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025aee lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x67a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.900050:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025aee lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x67a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.900059:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e8b497e6/0xd5401cbabc020829 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x681:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cd4 expref: 646 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.900063:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025aee lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x67a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.900091:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cba9880/2, rc 0, transno 4294975925 00010000:00010000:0.0:1783520935.900112:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e8b497e6/0xd5401cbabc020829 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x681:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cd4 expref: 646 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.900116:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.900120:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.900120:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993140864:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.900123:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.900157:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676a80 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.900157:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a80380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993142784:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.900209:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1783520935.900223:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470900 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.900224:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993142912:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.900270:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.900300:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993142912 00000100:00100000:1.0:1783520935.900304:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993142912 00000100:00100000:1.0:1783520935.900330:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44432 00000100:00100000:1.0:1783520935.900336:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cba9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+359:3627:x1870152993142912:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.900350:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1666 count 1 00010000:00010000:1.0:1783520935.900412:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025b26 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x682:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.900426:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000089851b12/0xd5401cbabc020837 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x682:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ce2 expref: 646 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.900434:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x682:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.900437:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x682:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.900441:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025b26 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x682:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.900465:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000089851b12/0xd5401cbabc020837 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x682:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ce2 expref: 646 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.900475:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000089851b12/0xd5401cbabc020837 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x682:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ce2 expref: 646 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.900485:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cbaa300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993143040:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.900540:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e240 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.900831:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943634571f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993141632:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.900847:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e473100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993142016:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.900851:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025afc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x67c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.900869:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025b0a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x67e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.900908:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.900924:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cbabb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993141504:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.900933:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.900945:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025af5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x67b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.900967:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002411392 00000100:00100000:2.0:1783520935.900972:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002411392 00000100:00100000:0.0:1783520935.900972:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002411264 00000100:00100000:0.0:1783520935.900986:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002411264 00010000:00010000:2.0:1783520935.901001:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008e368336/0xd5401cbabc0207ff lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x67b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c80 expref: 648 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.901018:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005fb589d4/0xd5401cbabc022a8f lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x67e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f317f expref: 648 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.901032:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36017 00000100:00100000:2.0:1783520935.901046:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634571f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+648:2387:x1870153002411392:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.901062:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.901075:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36018 00010000:00010000:2.0:1783520935.901076:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008e368336/0xd5401cbabc0207ff lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x67b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c80 expref: 648 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.901099:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634571c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+648:2387:x1870153002411264:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.901102:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008e368336/0xd5401cbabc0207ff lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x67b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c80 expref: 648 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.901122:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.901132:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008e368336/0xd5401cbabc0207ff lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x67b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c80 expref: 648 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.901137:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005fb589d4/0xd5401cbabc022a8f lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x67e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f317f expref: 648 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.901167:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x67b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.901171:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005fb589d4/0xd5401cbabc022a8f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x67e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f317f expref: 647 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.901174:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x67b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.901191:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025af5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x67b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.901201:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025af5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x67b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.901231:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005fb589d4/0xd5401cbabc022a8f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x67e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f317f expref: 647 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.901234:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.901260:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x67e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.901261:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025af5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x67b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.901268:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x67e:0x0].0x0 since=0s v=(1 0) 00000100:00100000:2.0:1783520935.901276:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634571f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+646:2387:x1870153002411392:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 227us (360us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.901290:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025b0a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x67e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.901296:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025af5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x67b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.901306:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36017 00010000:00010000:0.0:1783520935.901335:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025b0a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x67e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.901337:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025af5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x67b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.901354:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993141888:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.901421:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025af5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x67b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.901435:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025b03 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x67d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.901444:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.901447:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.901452:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.901459:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025af5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x67b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.901468:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.901473:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025af5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x67b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.901483:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025af5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x67b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.901490:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634571c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+645:2387:x1870153002411264:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 394us (554us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.901535:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36018 00010000:00010000:2.0:1783520935.901539:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025af5 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x67b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.901551:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025af5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x67b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.901572:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.901574:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.901578:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.901600:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516359180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+360:3625:x1870152993141120:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3996us (4333us total) trans 4294975926 rc 0/0 00010000:00010000:3.0:1783520935.901601:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025b0a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x67e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.901618:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025b0a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x67e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.901619:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44425 00010000:00010000:3.0:1783520935.901633:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025b0a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x67e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.901636:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351635b800/2, rc 0, transno 4294975926 00000100:00100000:2.0:1783520935.901647:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.901657:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025b0a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x67e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.901664:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351635b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993141120:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.901699:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520935.901717:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.901722:0:35284:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002411520 00000100:00100000:0.0:1783520935.901728:0:35284:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002411520 00000100:00100000:1.0:1783520935.901744:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1783520935.901762:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cbabb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993143168:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.901790:0:35284:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98411 00000100:00100000:0.0:1783520935.901804:0:35284:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634570700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_013:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+31:251805:x1870153002411520:12345-192.168.204.33@tcp:101:dd.0 00010000:00010000:0.0:1783520935.901829:0:35284:0:(ldlm_lockd.c:1265:ldlm_handle_enqueue()) ### server-side enqueue handler START 00000100:00100000:1.0:1783520935.901837:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.901839:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.901842:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d321180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+360:3627:x1870152993141760:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3638us (3768us total) trans 4294975927 rc 0/0 00000100:00100000:3.0:1783520935.901852:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44428 00010000:00010000:0.0:1783520935.901854:0:35284:0:(ldlm_lockd.c:1343:ldlm_handle_enqueue()) ### server-side enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbeba00/0xd5401cbabc025b2d lrc: 2/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT gid 0 flags: 0x40000000000000 nid: local remote: 0xc275876b859f34f8 expref: -99 pid: 35284 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.901887:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993143168 00000100:00100000:1.0:1783520935.901894:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993143168 00000004:00080000:3.0:1783520935.901895:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363d320a80/2, rc 0, transno 4294975927 00000100:00100000:2.0:1783520935.901926:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002411776 00000100:00100000:1.0:1783520935.901928:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44433 00000100:00100000:2.0:1783520935.901933:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002411776 00000100:00100000:1.0:1783520935.901940:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cba9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+360:3626:x1870152993143168:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.901945:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d320a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993141760:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.901955:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000078aff5f/0xd5401cbabc02080d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x67d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c9c expref: 645 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.901960:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1667 count 1 00010000:00010000:0.0:1783520935.901976:0:35284:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbeb200/0xd5401cbabc025b34 lrc: 3/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x40000000000000 pid: 35284 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.901982:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025b3b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x683:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.901987:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002411648 00000100:00100000:2.0:1783520935.901992:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002411648 00010000:00010000:0.0:1783520935.901996:0:35284:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbeb200/0xd5401cbabc025b34 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT flags: 0x50210000000000 pid: 35284 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.902002:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e473100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993143296:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.902012:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000813257ac/0xd5401cbabc020830 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x683:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cf0 expref: 646 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.902014:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b72c0a82/0xd5401cbabc020806 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x67c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c8e expref: 646 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.902031:0:35284:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbeb200/0xd5401cbabc025b34 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 35284 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.902055:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x683:0x0].0x0 since=37s v=(0 0) 00000100:00100000:3.0:1783520935.902061:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.902076:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x683:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.902083:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025b3b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x683:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.902088:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36019 00000100:00100000:2.0:1783520935.902102:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634573100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+646:2389:x1870153002411776:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520935.902107:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993143296 00000100:00100000:3.0:1783520935.902112:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993143296 00010000:00010000:2.0:1783520935.902114:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.902115:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36020 00010000:00010000:2.0:1783520935.902123:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000078aff5f/0xd5401cbabc02080d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x67d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c9c expref: 646 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.902131:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94361103b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+646:2389:x1870153002411648:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.902137:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000078aff5f/0xd5401cbabc02080d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x67d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c9c expref: 646 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.902144:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000004:00200000:0.0:1783520935.902149:0:35284:0:(mdt_handler.c:909:mdt_pack_attr2body()) [0x200000007:0x1:0x0]: returning size 69632 00010000:00010000:2.0:1783520935.902151:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000078aff5f/0xd5401cbabc02080d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x67d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c9c expref: 646 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.902155:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b72c0a82/0xd5401cbabc020806 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x67c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c8e expref: 646 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00200000:0.0:1783520935.902163:0:35284:0:(mdt_handler.c:1574:mdt_getattr_internal()) dirent count 0 stripe count 1 MDT count 2 00010000:00010000:2.0:1783520935.902185:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x67d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.902189:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x67d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.902193:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b72c0a82/0xd5401cbabc020806 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x67c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c8e expref: 645 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.902196:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025b03 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x67d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00010000:0.0:1783520935.902203:0:35284:0:(mdt_handler.c:2507:mdt_getattr_name_lock()) ### Returning lock to client ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbeb200/0xd5401cbabc025b34 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 35284 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.902239:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025b03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x67d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.902262:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b72c0a82/0xd5401cbabc020806 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x67c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0c8e expref: 645 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.902270:0:35284:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbeba00/0xd5401cbabc025b2d lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x44000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34f8 expref: 33 pid: 35284 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.902274:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.902298:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x67c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.902304:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x67c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.902306:0:35284:0:(ldlm_lockd.c:1513:ldlm_handle_enqueue()) ### server-side enqueue handler, sending reply (err=0, rc=0) ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbeb200/0xd5401cbabc025b34 lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x40200000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34f8 expref: 32 pid: 35284 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.902315:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025afc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x67c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.902319:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634573100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+644:2389:x1870153002411776:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 211us (597us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.902349:0:35284:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x0 (ffff943504b72c00) 00010000:00010000:1.0:1783520935.902351:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025afc lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x67c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.902353:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36019 00010000:00010000:0.0:1783520935.902411:0:35284:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 13 00010000:00010000:0.0:1783520935.902418:0:35284:0:(ldlm_lockd.c:1592:ldlm_handle_enqueue()) ### server-side enqueue handler END (lock ffff94351cbeb200, rc 0) 00010000:00010000:1.0:1783520935.902436:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.902458:0:35284:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634570700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_013:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+32:251805:x1870153002411520:12345-192.168.204.33@tcp:101:dd.0 Request processed in 655us (819us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.902467:0:35284:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98411 00000100:00100000:1.0:1783520935.902476:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94361103b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+643:2389:x1870153002411648:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 347us (644us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.902490:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36020 00010000:00010000:0.0:1783520935.902523:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.902532:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025b0a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x67e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.902543:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.902549:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025b0a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x67e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.902559:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cbaaa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993142528:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.902566:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025b0a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x67e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.902567:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943634573b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993142656:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.902583:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025b11 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x67f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.902597:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025b18 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x680:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.902619:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025b0a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x67e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.902633:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025b0a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x67e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.902639:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025b03 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x67d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.902646:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000813257ac/0xd5401cbabc020830 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x683:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cf0 expref: 642 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.902658:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.902660:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025b03 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x67d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.902665:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.902671:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.902673:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000813257ac/0xd5401cbabc020830 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x683:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cf0 expref: 642 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.902679:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025b03 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x67d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.902704:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025b03 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x67d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.902710:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cbaaa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993143424:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.902724:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.902739:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.902745:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025b03 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x67d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.902756:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e840 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.902759:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.902765:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025b03 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x67d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.902774:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025b03 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x67d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.902777:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025afc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x67c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.902792:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025afc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x67c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.902795:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025b03 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x67d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.902808:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025afc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x67c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.902810:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025b03 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x67d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.902829:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025afc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x67c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.902838:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.902842:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.902844:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to regular blp list, 0 blwis in pool 00010000:00010000:2.0:1783520935.902847:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025afc lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x67c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.902856:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.902859:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025afc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x67c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.902864:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44434 00010000:00010000:2.0:1783520935.902867:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025afc lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x67c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.902874:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e471f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+361:3626:x1870152993143296:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.902883:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025afc lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x67c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520935.902889:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1668 count 1 00010000:00010000:2.0:1783520935.902891:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025afc lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x67c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.902902:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025b42 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x684:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.902905:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.902907:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.902910:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.902913:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000dcb1462e/0xd5401cbabc02083e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x684:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cfe expref: 642 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.902917:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.902920:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x684:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.902924:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x684:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.902928:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025b42 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x684:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.902950:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516358000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+362:3626:x1870152993141248:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5148us (5267us total) trans 4294975928 rc 0/0 00000100:00100000:1.0:1783520935.902958:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44426 00000004:00080000:3.0:1783520935.902961:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351635a680/2, rc 0, transno 4294975928 00000100:00100000:3.0:1783520935.902973:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351635a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993141248:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.903002:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516359f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+362:3627:x1870152993141376:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5081us (5176us total) trans 4294975929 rc 0/0 00000100:00100000:2.0:1783520935.903010:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44427 00000100:00100000:1.0:1783520935.903027:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.903028:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e470e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993143552:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.903054:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002411904 00000100:00100000:2.0:1783520935.903058:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002411904 00000100:00100000:3.0:1783520935.903063:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.903084:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000906d822b/0xd5401cbabc02081b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x67f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cb8 expref: 643 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.903117:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351635b100/2, rc 0, transno 4294975929 00000100:00100000:1.0:1783520935.903119:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.903121:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36021 00000100:00100000:3.0:1783520935.903128:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351635b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993141376:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.903138:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a80380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993142784:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.903139:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94361b4a5850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+643:2388:x1870153002411904:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.903171:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.903178:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000906d822b/0xd5401cbabc02081b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x67f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cb8 expref: 643 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.903186:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025b1f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x681:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.903196:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.903199:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000906d822b/0xd5401cbabc02081b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x67f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cb8 expref: 643 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.903200:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993143552 00000100:00100000:3.0:1783520935.903206:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993143552 00000100:00100000:3.0:1783520935.903250:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44435 00010000:00010000:2.0:1783520935.903250:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000906d822b/0xd5401cbabc02081b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x67f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cb8 expref: 643 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.903260:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e471c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+362:3627:x1870152993143552:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.903266:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x67f:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.903266:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e472680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993143680:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.903274:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00002000:00080000:3.0:1783520935.903275:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1669 count 1 00010000:00010000:2.0:1783520935.903281:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x67f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.903290:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025b11 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x67f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.903298:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025b49 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x685:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.903301:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025b11 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x67f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.903305:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002412032 00010000:00010000:3.0:1783520935.903312:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000fa3b2033/0xd5401cbabc020845 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x685:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d0c expref: 642 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.903312:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002412032 00010000:00010000:2.0:1783520935.903325:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.903334:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001daa4832/0xd5401cbabc020822 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x680:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cc6 expref: 643 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.903353:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.903381:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94361b4a5850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+643:2388:x1870153002411904:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 224us (331us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.903386:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x685:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.903390:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x685:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.903392:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36021 00010000:00010000:3.0:1783520935.903394:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025b49 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x685:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.903408:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002412160 00000100:00100000:1.0:1783520935.903414:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002412160 00010000:00010000:3.0:1783520935.903425:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000fa3b2033/0xd5401cbabc020845 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x685:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d0c expref: 643 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.903436:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e8b497e6/0xd5401cbabc020829 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x681:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cd4 expref: 643 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.903441:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000fa3b2033/0xd5401cbabc020845 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x685:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d0c expref: 643 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.903459:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002412288 00000100:00100000:2.0:1783520935.903464:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002412288 00000100:00100000:3.0:1783520935.903465:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e470000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993143808:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.903478:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36022 00010000:00010000:2.0:1783520935.903485:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000089851b12/0xd5401cbabc020837 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x682:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ce2 expref: 644 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.903492:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36023 00000100:00100000:0.0:1783520935.903516:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cbab800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+644:2387:x1870153002412032:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520935.903524:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435129679c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.903528:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36024 00000100:00100000:1.0:1783520935.903528:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cbab480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+644:2390:x1870153002412160:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.903534:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.903543:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520935.903546:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cbaa680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+644:2388:x1870153002412288:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.903547:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001daa4832/0xd5401cbabc020822 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x680:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cc6 expref: 644 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.903556:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e8b497e6/0xd5401cbabc020829 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x681:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cd4 expref: 644 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.903562:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000dcb1462e/0xd5401cbabc02083e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x684:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cfe expref: 644 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.903570:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.903584:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000089851b12/0xd5401cbabc020837 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x682:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ce2 expref: 644 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.903609:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001daa4832/0xd5401cbabc020822 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x680:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cc6 expref: 644 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.903613:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e8b497e6/0xd5401cbabc020829 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x681:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cd4 expref: 644 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.903615:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000dcb1462e/0xd5401cbabc02083e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x684:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cfe expref: 644 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.903622:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000089851b12/0xd5401cbabc020837 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x682:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ce2 expref: 644 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.903679:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e473b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993143936:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.903689:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x682:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.903691:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001daa4832/0xd5401cbabc020822 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x680:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cc6 expref: 644 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.903693:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e8b497e6/0xd5401cbabc020829 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x681:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cd4 expref: 644 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.903695:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x682:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.903704:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967e40 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.903720:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025b26 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x682:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.903730:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993143680 00000100:00100000:3.0:1783520935.903740:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993143680 00010000:00010000:0.0:1783520935.903743:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x680:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.903755:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x681:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.903760:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x680:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.903763:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025b26 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x682:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.903766:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x681:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.903770:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025b18 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x680:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.903772:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44436 00010000:00010000:1.0:1783520935.903783:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025b1f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x681:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.903793:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943620485850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+363:3628:x1870152993143680:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.903824:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.903833:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025b18 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x680:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.903835:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025b1f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x681:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520935.903837:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1670 count 1 00000100:00100000:2.0:1783520935.903851:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cbaa680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+642:2388:x1870153002412288:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 307us (577us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.903857:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025b50 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x686:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.903891:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36024 00010000:00010000:3.0:1783520935.903902:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000039ed8f91/0xd5401cbabc02297e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x686:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f306e expref: 642 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.903911:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x686:0x0].0x0 since=36s v=(0 0) 00010000:00010000:0.0:1783520935.903911:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.903919:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x686:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.903920:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.903924:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025b50 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x686:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.903946:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cbab800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+641:2387:x1870153002412032:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 452us (829us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.903954:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000039ed8f91/0xd5401cbabc02297e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x686:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f306e expref: 641 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.903958:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cbab480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+641:2390:x1870153002412160:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 432us (766us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.903985:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36022 00010000:00010000:3.0:1783520935.903999:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000039ed8f91/0xd5401cbabc02297e lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x686:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f306e expref: 641 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.903999:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36023 00000100:00100000:3.0:1783520935.904019:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e473480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993144064:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.904038:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025b1f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x681:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.904050:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff9436298a6480 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.904051:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025b1f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x681:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.904060:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025b11 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x67f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.904069:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025b1f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x681:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.904107:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025b1f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x681:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.904108:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025b11 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x67f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.904137:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025b11 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x67f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.904140:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.904154:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025b11 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x67f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.904155:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.904165:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025b1f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x681:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.904173:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.904177:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025b1f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x681:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.904186:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025b1f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x681:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.904187:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.904199:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.904210:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025b11 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x67f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.904217:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025b1f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x681:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.904226:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.904235:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025b11 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x67f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.904236:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025b1f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x681:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.904275:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025b11 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x67f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.904296:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a81880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+364:3626:x1870152993142144:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4676us (5045us total) trans 4294975930 rc 0/0 00010000:00010000:0.0:1783520935.904298:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025b11 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x67f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.904303:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.904307:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.904315:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520935.904316:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943509a80700/2, rc 0, transno 4294975930 00000100:00100000:1.0:1783520935.904328:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44429 00000100:00100000:3.0:1783520935.904337:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a80700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993142144:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.904344:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025b11 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x67f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.904416:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.904419:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.904422:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.904430:0:9151:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000089851b12/0xd5401cbabc020837 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x682:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ce2 expref: 639 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.904448:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cbaa300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993143040:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.904471:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025b26 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x682:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.904480:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025b26 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x682:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.904485:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025b26 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x682:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.904491:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025b26 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x682:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.904512:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025b26 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x682:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.904524:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.904525:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94361103b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993144192:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.904539:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.904547:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025b26 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x682:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.904556:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.904561:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025b26 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x682:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.904571:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025b26 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x682:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.904582:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.904601:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025b26 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x682:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.904610:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025b26 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x682:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.904636:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cba9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+365:3627:x1870152993142912:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4296us (4361us total) trans 4294975932 rc 0/0 00010000:00010000:2.0:1783520935.904644:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.904649:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.904650:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44432 00010000:00010000:2.0:1783520935.904652:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.904670:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993144192 00000100:00100000:1.0:1783520935.904675:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993144192 00000004:00080000:3.0:1783520935.904694:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cba9f80/2, rc 0, transno 4294975932 00000100:00100000:3.0:1783520935.904712:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993142912:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.904720:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a81500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+365:3628:x1870152993142400:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4913us (5074us total) trans 4294975931 rc 0/0 00000100:00100000:1.0:1783520935.904727:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44437 00000100:00100000:0.0:1783520935.904750:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44431 00000100:00100000:1.0:1783520935.904751:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a80380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+365:3625:x1870152993144192:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.904757:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943634571180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993144320:0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.904770:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1671 count 1 00010000:00010000:1.0:1783520935.904788:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025b57 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x687:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.904802:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004bf7a5b2/0xd5401cbabc020853 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x687:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d28 expref: 638 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.904812:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943509a82d80/2, rc 0, transno 4294975931 00000100:00100000:3.0:1783520935.904813:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.904813:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x687:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.904826:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x687:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.904831:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025b57 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x687:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.904838:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a82d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993142400:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.904851:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993144320 00000100:00100000:1.0:1783520935.904859:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993144320 00010000:00010000:0.0:1783520935.904894:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025b18 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x680:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.904906:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025b18 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x680:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.904908:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943634571f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993144448:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.904911:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44438 00000100:00100000:1.0:1783520935.904930:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e471500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+365:3626:x1870152993144320:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.904949:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00080000:1.0:1783520935.904949:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1672 count 1 00010000:00010000:0.0:1783520935.904952:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025b18 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x680:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.904961:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025b5e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x688:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.904965:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025b18 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x680:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.904980:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000018da641d/0xd5401cbabc02085a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x688:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d36 expref: 638 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.904991:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x688:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.904995:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x688:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.904999:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025b5e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x688:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.905007:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993144448 00000100:00100000:0.0:1783520935.905014:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993144448 00010000:00010000:1.0:1783520935.905026:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000018da641d/0xd5401cbabc02085a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x688:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d36 expref: 638 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.905041:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000018da641d/0xd5401cbabc02085a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x688:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d36 expref: 638 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.905050:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44439 00000100:00100000:1.0:1783520935.905059:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cbaa300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993144576:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.905067:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634571c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+366:3627:x1870152993144448:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.905092:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1673 count 1 00000100:00100000:1.0:1783520935.905106:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.905113:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025b65 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x689:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.905132:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004f93cd17/0xd5401cbabc020861 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x689:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d44 expref: 638 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.905139:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004bf7a5b2/0xd5401cbabc020853 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x687:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d28 expref: 638 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.905163:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x689:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.905169:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004bf7a5b2/0xd5401cbabc020853 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x687:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d28 expref: 638 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.905174:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x689:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.905185:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba8380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993144704:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.905186:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025b65 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x689:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.905212:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e780 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.905224:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004f93cd17/0xd5401cbabc020861 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x689:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d44 expref: 638 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.905236:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004f93cd17/0xd5401cbabc020861 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x689:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d44 expref: 638 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.905246:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a82680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993144832:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.905288:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470e40 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.905328:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.905341:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.905349:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025b18 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x680:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.905427:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.905430:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025b18 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x680:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.905438:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025b18 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x680:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.905449:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025b18 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x680:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.905454:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025b18 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x680:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.905474:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.905477:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.905480:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.905534:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a83b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+367:3625:x1870152993142272:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5798us (6005us total) trans 4294975933 rc 0/0 00000100:00100000:0.0:1783520935.905540:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44430 00000004:00080000:0.0:1783520935.905570:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943509a82a00/2, rc 0, transno 4294975933 00000100:00100000:0.0:1783520935.905580:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a82a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993142272:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.905676:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943634573100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993144960:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.905699:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.905789:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993144960 00000100:00100000:1.0:1783520935.905797:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993144960 00000100:00100000:1.0:1783520935.905821:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44440 00000100:00100000:1.0:1783520935.905826:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a80e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+367:3628:x1870152993144960:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.905841:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1674 count 1 00010000:00010000:1.0:1783520935.905855:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025b6c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x68a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.905869:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ec56d795/0xd5401cbabc020868 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x68a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d52 expref: 638 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.905882:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x68a:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.905886:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x68a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.905890:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025b6c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x68a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.905911:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ec56d795/0xd5401cbabc020868 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x68a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d52 expref: 638 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.905921:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ec56d795/0xd5401cbabc020868 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x68a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d52 expref: 638 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.905933:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cbaad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993145088:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.905968:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e600 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.906588:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e470000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993143808:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.906624:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025b49 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x685:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.906638:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cbaaa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993143424:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.906657:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025b3b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x683:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.906705:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e473480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993144064:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.906715:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.906723:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025b50 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x686:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.906743:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.906752:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e473b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993143936:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.906764:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002412416 00000100:00100000:2.0:1783520935.906769:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002412416 00010000:00010000:3.0:1783520935.906772:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025b42 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x684:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.906793:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000813257ac/0xd5401cbabc020830 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x683:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cf0 expref: 639 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.906816:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002412544 00000100:00100000:2.0:1783520935.906819:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002412544 00010000:00010000:2.0:1783520935.906832:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000fa3b2033/0xd5401cbabc020845 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x685:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d0c expref: 640 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.906865:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36025 00000100:00100000:2.0:1783520935.906874:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634573b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+640:2389:x1870153002412416:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.906884:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.906890:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000813257ac/0xd5401cbabc020830 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x683:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cf0 expref: 640 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.906902:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000813257ac/0xd5401cbabc020830 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x683:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cf0 expref: 640 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.906918:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000813257ac/0xd5401cbabc020830 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x683:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cf0 expref: 640 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.906931:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x683:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.906935:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x683:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.906940:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025b3b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x683:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.906941:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36026 00010000:00010000:2.0:1783520935.906953:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025b3b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x683:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.906957:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515776680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+639:2389:x1870153002412544:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.906970:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cbaa300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993144576:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.906973:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.906982:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000fa3b2033/0xd5401cbabc020845 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x685:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d0c expref: 639 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.906984:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.906989:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025b5e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x688:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.907003:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000fa3b2033/0xd5401cbabc020845 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x685:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d0c expref: 639 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.907019:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634573b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+639:2389:x1870153002412416:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 144us (304us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.907022:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000fa3b2033/0xd5401cbabc020845 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x685:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d0c expref: 639 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.907034:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36025 00010000:00010000:3.0:1783520935.907048:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x685:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.907053:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x685:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.907057:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025b49 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x685:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.907064:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025b3b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x683:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.907067:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025b49 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x685:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.907086:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025b3b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x683:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.907096:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025b3b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x683:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.907099:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.907107:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025b3b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x683:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.907114:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025b49 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x685:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.907121:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025b49 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x685:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.907124:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.907137:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.907141:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025b3b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x683:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.907149:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.907153:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025b3b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x683:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.907162:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025b3b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x683:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.907174:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025b3b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x683:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.907182:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025b3b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x683:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.907199:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.907202:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.907205:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.907222:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.907238:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025b49 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x685:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.907247:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025b49 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x685:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.907261:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.907309:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.907345:0:6495:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002413056 00000100:00100000:0.0:1783520935.907354:0:6495:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002413056 00000100:00100000:2.0:1783520935.907413:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cba9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+369:3626:x1870152993143168:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5471us (5580us total) trans 4294975934 rc 0/0 00000100:00100000:1.0:1783520935.907422:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.907427:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44433 00000100:00100000:0.0:1783520935.907446:0:6495:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98412 00000100:00100000:3.0:1783520935.907458:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e471c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+368:3627:x1870152993143552:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4196us (4390us total) trans 4294975935 rc 0/0 00000100:00100000:1.0:1783520935.907463:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002412928 00000100:00100000:2.0:1783520935.907468:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002412672 00000100:00100000:1.0:1783520935.907469:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002412928 00000100:00100000:3.0:1783520935.907477:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44435 00000100:00100000:0.0:1783520935.907477:0:6495:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cbaa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+32:251805:x1870153002413056:12345-192.168.204.33@tcp:101:dd.0 00000100:00100000:2.0:1783520935.907480:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002412672 00010000:00010000:1.0:1783520935.907516:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000dcb1462e/0xd5401cbabc02083e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x684:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cfe expref: 639 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.907517:0:6495:0:(ldlm_lockd.c:1265:ldlm_handle_enqueue()) ### server-side enqueue handler START 00010000:00010000:2.0:1783520935.907519:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000039ed8f91/0xd5401cbabc02297e lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x686:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f306e expref: 639 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.907524:0:6495:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 2 locks, starting at 1 00000004:00080000:3.0:1783520935.907543:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cbabb80/2, rc 0, transno 4294975934 00000100:00100000:3.0:1783520935.907562:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cbabb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993143168:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.907562:0:6495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbeb200/0xd5401cbabc025b34 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x44a01000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34f8 expref: 32 pid: 35284 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.907566:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36027 00000100:00100000:1.0:1783520935.907590:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cbaaa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+639:2388:x1870153002412928:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.907605:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000004:00080000:3.0:1783520935.907606:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e470e00/2, rc 0, transno 4294975935 00010000:00010000:0.0:1783520935.907613:0:6495:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x0 (ffff943504b72c00) 00010000:00010000:1.0:1783520935.907615:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000dcb1462e/0xd5401cbabc02083e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x684:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cfe expref: 639 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.907622:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e470e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993143552:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.907633:0:6495:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.907649:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000dcb1462e/0xd5401cbabc02083e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x684:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cfe expref: 639 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.907656:0:6495:0:(ldlm_lockd.c:1343:ldlm_handle_enqueue()) ### server-side enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbeac00/0xd5401cbabc025b73 lrc: 2/0,0 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT gid 0 flags: 0x40000000000000 nid: local remote: 0xc275876b859f34ff expref: -99 pid: 6495 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.907657:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36028 00000100:00100000:3.0:1783520935.907664:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e007b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993145344:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.907688:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000dcb1462e/0xd5401cbabc02083e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x684:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0cfe expref: 639 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.907689:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94361b4a4050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+639:2388:x1870153002412672:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.907710:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.907713:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x684:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.907719:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000039ed8f91/0xd5401cbabc02297e lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x686:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f306e expref: 638 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.907720:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x684:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.907723:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.907728:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025b42 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x684:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.907744:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025b42 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x684:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.907745:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000039ed8f91/0xd5401cbabc02297e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x686:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f306e expref: 638 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.907754:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1783520935.907765:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515777100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993145216:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.907779:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000039ed8f91/0xd5401cbabc02297e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x686:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f306e expref: 638 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.907794:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x686:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.907794:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.907800:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.907803:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x686:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.907808:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025b50 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x686:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.907816:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025b50 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x686:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.907822:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cbaaa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+637:2388:x1870153002412928:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 240us (512us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.907832:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36027 00010000:00010000:2.0:1783520935.907836:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.907837:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025b42 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x684:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.907849:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025b42 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x684:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.907858:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025b42 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x684:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.907858:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993145344 00000100:00100000:1.0:1783520935.907864:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993145344 00000100:00100000:2.0:1783520935.907865:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94361b4a4050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+636:2388:x1870153002412672:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 185us (448us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.907881:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025b42 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x684:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.907882:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36028 00000100:00100000:1.0:1783520935.907889:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993145216 00000100:00100000:1.0:1783520935.907894:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993145216 00010000:00010000:3.0:1783520935.907908:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520935.907918:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00000100:00100000:1.0:1783520935.907924:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44441 00010000:00010000:2.0:1783520935.907927:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025b49 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x685:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.907936:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e473b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+369:3626:x1870152993145344:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.907940:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.907946:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025b49 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x685:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.907955:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025b49 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x685:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.907960:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1676 count 1 00010000:00010000:0.0:1783520935.907967:0:6495:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbebe00/0xd5401cbabc025b7a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x40000000000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.907972:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025b49 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x685:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.907982:0:6495:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbebe00/0xd5401cbabc025b7a lrc: 3/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210000000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.907984:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025b81 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x68c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.907986:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025b49 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x685:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.908011:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e471f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+370:3626:x1870152993143296:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5132us (5958us total) trans 4294975936 rc 0/0 00010000:00010000:0.0:1783520935.908017:0:6495:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbebe00/0xd5401cbabc025b7a lrc: 3/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.908042:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44434 00010000:00010000:1.0:1783520935.908044:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000008908ae1/0xd5401cbabc020876 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x68c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d6e expref: 635 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.908056:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x68c:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.908059:0:6495:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbeb800/0xd5401cbabc025b88 lrc: 3/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x40000000000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.908061:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x68c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.908064:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.908075:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025b42 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x684:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.908079:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e473100/2, rc 0, transno 4294975936 00010000:00010000:1.0:1783520935.908079:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025b81 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x68c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.908102:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e473100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993143296:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.908128:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.908129:0:6495:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbeb800/0xd5401cbabc025b88 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.908139:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025b42 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x684:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.908144:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44442 00010000:00010000:0.0:1783520935.908146:0:6495:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbeb800/0xd5401cbabc025b88 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210000000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.908149:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000008908ae1/0xd5401cbabc020876 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x68c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d6e expref: 635 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.908163:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e473480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+369:3626:x1870152993145216:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.908173:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025b42 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x684:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520935.908198:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1675 count 1 00010000:00010000:1.0:1783520935.908199:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000008908ae1/0xd5401cbabc020876 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x68c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d6e expref: 635 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.908214:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025b42 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x684:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.908217:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025b8f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x68b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.908218:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993145472:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.908237:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025b42 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x684:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.908244:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e86247cd/0xd5401cbabc02086f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x68b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d60 expref: 635 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.908253:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x68b:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.908256:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.908259:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x68b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.908261:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.908265:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025b8f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x68b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.908268:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.908270:0:6495:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbea000/0xd5401cbabc025b96 lrc: 3/1,0 mode: --/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x0/0x0 rrc: 7 type: IBT flags: 0x40000000000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.908272:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.908303:0:6495:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbea000/0xd5401cbabc025b96 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 7 type: IBT flags: 0x50210000000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.908313:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e86247cd/0xd5401cbabc02086f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x68b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d60 expref: 635 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.908327:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.908327:0:6495:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbea000/0xd5401cbabc025b96 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 7 type: IBT flags: 0x40210000000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.908343:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e86247cd/0xd5401cbabc02086f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x68b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d60 expref: 635 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00200000:0.0:1783520935.908354:0:6495:0:(mdt_handler.c:909:mdt_pack_attr2body()) [0x200000402:0x72a:0x0]: returning size 131072 00000100:00100000:2.0:1783520935.908403:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002413184 00000100:00100000:3.0:1783520935.908408:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e470380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993145728:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.908410:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002413184 00000100:00100000:3.0:1783520935.908434:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e007480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993145600:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.908465:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000018da641d/0xd5401cbabc02085a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x688:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d36 expref: 636 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.908483:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.908510:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36029 00010000:00010000:0.0:1783520935.908518:0:6495:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbebe00/0xd5401cbabc025b7a lrc: 4/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.908525:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516358700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+636:2387:x1870153002413184:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.908531:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cbaad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993145088:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.908552:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.908552:0:6495:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbebe00/0xd5401cbabc025b7a lrc: 4/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210400000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.908563:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943604fbc840 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.908567:0:6495:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbebe00/0xd5401cbabc025b7a lrc: 3/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x50210400000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.908571:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000018da641d/0xd5401cbabc02085a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x688:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d36 expref: 636 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.908571:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025b6c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x68a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.908596:0:6495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbebe00/0xd5401cbabc025b7a lrc: 4/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210400000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.908599:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515776680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+636:2389:x1870153002412544:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 1639us (1850us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.908631:0:6495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbebe00 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:0.0:1783520935.908638:0:6495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbebe00/0xd5401cbabc025b7a lrc: 4/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40210400000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.908646:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000018da641d/0xd5401cbabc02085a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x688:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d36 expref: 636 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.908651:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36026 00000100:00100000:1.0:1783520935.908664:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba8380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993144704:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.908690:0:6495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbebe00/0xd5401cbabc025b7a lrc: 5/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x40218400000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.908699:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025b57 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x687:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.908700:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000018da641d/0xd5401cbabc02085a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x688:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d36 expref: 635 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.908716:0:6495:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x0 (ffff943504b72c00) 00010000:00010000:3.0:1783520935.908718:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025b50 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x686:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.908731:0:6495:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:0.0:1783520935.908741:0:6495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbebe00/0xd5401cbabc025b7a lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x44a19400000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.908766:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025b50 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x686:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.908770:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x688:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.908770:0:6495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbebe00/0xd5401cbabc025b7a lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x44a19400000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.908787:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x688:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.908813:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025b5e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x688:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.908816:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025b50 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x686:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.908824:0:6495:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbeb800/0xd5401cbabc025b88 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.908836:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025b5e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x688:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.908861:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025b50 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x686:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.908863:0:6495:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbeb800/0xd5401cbabc025b88 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210400000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.908893:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.908902:0:6495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbeb800/0xd5401cbabc025b88 lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.908911:0:6495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb800 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:3.0:1783520935.908917:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.908917:0:6495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbeb800/0xd5401cbabc025b88 lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.908929:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516358700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+634:2387:x1870153002413184:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 406us (605us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.908948:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025b5e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x688:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.908950:0:6495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbeb800/0xd5401cbabc025b88 lrc: 4/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40218400000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.908956:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36029 00010000:00010000:0.0:1783520935.908980:0:6495:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x5ba6781 (ffff943515473400) 00010000:00010000:1.0:1783520935.908981:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025b5e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x688:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.908985:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.908991:0:6495:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:2.0:1783520935.908996:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025b50 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x686:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.908998:0:6495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbeb800/0xd5401cbabc025b88 lrc: 2/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.909008:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025b5e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x688:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.909055:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.909064:0:6495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbeb800/0xd5401cbabc025b88 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 6495 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.909068:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025b5e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x688:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.909069:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025b50 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x686:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.909096:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025b50 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x686:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.909115:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943620485850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+370:3628:x1870152993143680:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5324us (5768us total) trans 4294975937 rc 0/0 00010000:00010000:0.0:1783520935.909120:0:6495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbeac00/0xd5401cbabc025b73 lrc: 0/0,0 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x44000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34ff expref: 34 pid: 6495 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.909126:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025b50 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x686:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.909128:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520935.909143:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44436 00010000:00010000:0.0:1783520935.909165:0:6495:0:(ldlm_lockd.c:1513:ldlm_handle_enqueue()) ### server-side enqueue handler, sending reply (err=0, rc=0) ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbea000/0xd5401cbabc025b96 lrc: 3/0,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 7 type: IBT gid 0 flags: 0x40200000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34ff expref: 33 pid: 6495 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.909168:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025b50 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x686:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.909184:0:6495:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x72a:0x0].0x0 (ffff943515473e00) 00010000:00010000:0.0:1783520935.909196:0:6495:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 1b 00010000:00010000:0.0:1783520935.909202:0:6495:0:(ldlm_lockd.c:1592:ldlm_handle_enqueue()) ### server-side enqueue handler END (lock ffff94351cbea000, rc 0) 00010000:00010000:2.0:1783520935.909237:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.909243:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025b5e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x688:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.909253:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.909258:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025b5e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x688:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.909258:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e471500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+370:3626:x1870152993144320:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4330us (4448us total) trans 4294975938 rc 0/0 00000100:00100000:0.0:1783520935.909266:0:6495:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cbaa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+33:251805:x1870153002413056:12345-192.168.204.33@tcp:101:dd.0 Request processed in 1803us (2038us total) trans 8589957902 rc 0/0 00000100:00100000:1.0:1783520935.909296:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44438 00010000:00010000:2.0:1783520935.909302:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025b5e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x688:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.909303:0:6495:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98412 00000100:00100000:1.0:1783520935.909329:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993145600 00010000:00010000:2.0:1783520935.909331:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025b5e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x688:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.909337:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993145600 00010000:00010000:2.0:1783520935.909341:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025b5e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x688:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.909428:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.909449:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44443 00000100:00100000:1.0:1783520935.909468:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e472300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+370:3627:x1870152993145600:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.909487:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1677 count 1 00010000:00010000:1.0:1783520935.909519:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025ba4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x68d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.909532:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000acb12ba4/0xd5401cbabc02087d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x68d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d7c expref: 633 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.909533:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520935.909538:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e472680/2, rc 0, transno 4294975937 00010000:00010000:2.0:1783520935.909540:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.909551:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x68d:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.909552:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.909558:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x68d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.909564:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025ba4 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x68d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.909571:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.909573:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e472680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993143680:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.909580:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.909586:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520935.909588:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943634571180/2, rc 0, transno 4294975938 00000100:00100000:3.0:1783520935.909599:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943634571180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993144320:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.909604:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000acb12ba4/0xd5401cbabc02087d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x68d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d7c expref: 633 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.909618:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000acb12ba4/0xd5401cbabc02087d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x68d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d7c expref: 633 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.909637:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351635aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993145856:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.909645:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.909694:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.909697:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e600 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.909703:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1783520935.909721:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a80000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993145984:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.909734:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002413440 00000100:00100000:1.0:1783520935.909739:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002413440 00000100:00100000:2.0:1783520935.909741:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002413312 00000100:00100000:0.0:1783520935.909749:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a83800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993146112:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.909750:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002413312 00010000:00010000:1.0:1783520935.909762:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004f93cd17/0xd5401cbabc020861 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x689:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d44 expref: 634 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.909769:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.909786:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004bf7a5b2/0xd5401cbabc020853 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x687:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d28 expref: 635 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.909796:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36030 00000100:00100000:1.0:1783520935.909808:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e006d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+635:2387:x1870153002413440:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.909813:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.909835:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36031 00000100:00100000:1.0:1783520935.909846:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508059f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+635:2388:x1870153002413312:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.909851:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.909857:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004bf7a5b2/0xd5401cbabc020853 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x687:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d28 expref: 635 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.909866:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004bf7a5b2/0xd5401cbabc020853 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x687:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d28 expref: 635 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.909881:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004bf7a5b2/0xd5401cbabc020853 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x687:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d28 expref: 635 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.909900:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x687:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.909902:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x687:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.909907:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025b57 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x687:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.909914:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025b57 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x687:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.909917:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993145984 00000100:00100000:0.0:1783520935.909925:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993145984 00000100:00100000:2.0:1783520935.909932:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.909938:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.909974:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508059f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+634:2388:x1870153002413312:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 125us (279us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.909974:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993146112 00000100:00100000:0.0:1783520935.909981:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993146112 00000100:00100000:1.0:1783520935.909989:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36031 00000100:00100000:3.0:1783520935.909991:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44444 00000100:00100000:3.0:1783520935.910009:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e471c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+372:3627:x1870152993145984:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.910019:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002413568 00000100:00100000:1.0:1783520935.910025:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002413568 00002000:00080000:3.0:1783520935.910029:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1678 count 1 00000100:00100000:0.0:1783520935.910031:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44445 00010000:00010000:1.0:1783520935.910047:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ec56d795/0xd5401cbabc020868 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x68a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d52 expref: 634 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.910050:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a83100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+372:3625:x1870152993146112:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.910051:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025bab lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x68e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.910090:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1679 count 1 00010000:00010000:3.0:1783520935.910092:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a2630d1b/0xd5401cbabc02286d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x68e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f5d expref: 634 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.910105:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x68e:0x0].0x0 since=36s v=(0 0) 00010000:00010000:3.0:1783520935.910109:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x68e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.910113:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025bb2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x68f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.910114:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36032 00010000:00010000:3.0:1783520935.910118:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025bab lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x68e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.910140:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520935.910142:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350805b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+634:2388:x1870153002413568:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.910161:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004f93cd17/0xd5401cbabc020861 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x689:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d44 expref: 634 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.910164:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.910168:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f48ba41a/0xd5401cbabc02088b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x68f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d98 expref: 634 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.910176:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ec56d795/0xd5401cbabc020868 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x68a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d52 expref: 634 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.910218:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x68f:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.910235:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x68f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.910237:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004f93cd17/0xd5401cbabc020861 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x689:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d44 expref: 634 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.910244:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ec56d795/0xd5401cbabc020868 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x68a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d52 expref: 634 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.910244:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025bb2 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x68f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.910254:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a2630d1b/0xd5401cbabc02286d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x68e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f5d expref: 634 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.910298:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x689:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.910306:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x689:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.910310:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025b65 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x689:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.910312:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a2630d1b/0xd5401cbabc02286d lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x68e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f5d expref: 634 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.910321:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ec56d795/0xd5401cbabc020868 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x68a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d52 expref: 634 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.910323:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f48ba41a/0xd5401cbabc02088b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x68f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d98 expref: 634 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.910383:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025b65 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x689:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.910409:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f48ba41a/0xd5401cbabc02088b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x68f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d98 expref: 633 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.910410:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x68a:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.910424:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e471f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993146240:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.910435:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x68a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.910436:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.910456:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025b6c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.910466:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a81c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993146368:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.910471:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025b6c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x68a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.910478:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e006d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+633:2387:x1870153002413440:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 666us (830us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.910492:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8240 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.910507:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36030 00000100:00100000:0.0:1783520935.910539:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470b40 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.910553:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.910557:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025b57 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x687:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.910567:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025b57 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x687:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.910578:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025b57 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x687:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.910585:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350805b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+632:2388:x1870153002413568:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 454us (657us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.910590:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025b57 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x687:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.910597:0:35282:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004f93cd17/0xd5401cbabc020861 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x689:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d44 expref: 632 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.910600:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36032 00000100:00100000:0.0:1783520935.910644:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a82680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993144832:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.910654:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.910672:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.910680:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025b57 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x687:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.910683:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025b65 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x689:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.910719:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.910724:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025b65 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x689:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.910728:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025b57 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x687:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.910749:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025b65 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x689:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.910767:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025b57 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x687:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.910781:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025b65 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x689:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.910788:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025b57 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x687:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.910795:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a80380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+373:3625:x1870152993144192:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6045us (6212us total) trans 4294975939 rc 0/0 00010000:00010000:0.0:1783520935.910829:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025b65 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x689:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.910831:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025b57 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x687:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.910838:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44437 00010000:00010000:1.0:1783520935.910875:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025b6c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.910892:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025b6c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.910895:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.910900:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.910904:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025b6c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.910906:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520935.910911:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94361103b100/2, rc 0, transno 4294975939 00010000:00010000:1.0:1783520935.910919:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025b6c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x68a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.910932:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94361103b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993144192:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.910941:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.911016:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.911025:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025b6c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x68a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.911035:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbbc00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.911040:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025b6c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x68a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.911047:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a80e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+373:3628:x1870152993144960:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5212us (5338us total) trans 4294975940 rc 0/0 00010000:00010000:0.0:1783520935.911065:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025b6c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x68a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.911071:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44440 00010000:00010000:0.0:1783520935.911106:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025b6c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x68a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.911114:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006e2a8486/0xd5401cbabc025b6c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x68a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.911141:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.911144:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.911148:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.911166:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a82680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993146496:0@lo:6:osp-syn-1-0.0 00000004:00080000:0.0:1783520935.911191:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943634573100/2, rc 0, transno 4294975940 00000100:00100000:0.0:1783520935.911205:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943634573100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993144960:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.911217:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.911267:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993146496 00000100:00100000:3.0:1783520935.911274:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993146496 00000100:00100000:0.0:1783520935.911316:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a81f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993146624:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.911319:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44446 00000100:00100000:3.0:1783520935.911337:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e470000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+373:3626:x1870152993146496:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520935.911419:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1680 count 1 00000100:00100000:0.0:1783520935.911434:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.911469:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993146624 00000100:00100000:1.0:1783520935.911476:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993146624 00010000:00010000:0.0:1783520935.911529:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.911530:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44447 00000100:00100000:1.0:1783520935.911548:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a83480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+374:3627:x1870152993146624:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.911551:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.911559:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025b65 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x689:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.911568:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1681 count 1 00010000:00010000:2.0:1783520935.911571:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.911579:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025b65 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x689:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.911588:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025bb9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x691:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.911598:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025b65 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x689:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.911604:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000346e8ac0/0xd5401cbabc0208a0 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x691:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0db4 expref: 630 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.911627:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x691:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.911634:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x691:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.911635:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025b65 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x689:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.911642:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025bb9 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x691:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.911649:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025b65 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x689:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.911685:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000346e8ac0/0xd5401cbabc0208a0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x691:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0db4 expref: 630 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.911704:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000346e8ac0/0xd5401cbabc0208a0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x691:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0db4 expref: 630 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.911712:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.911716:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634571c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+375:3627:x1870152993144448:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6648us (6764us total) trans 4294975941 rc 0/0 00000100:00100000:1.0:1783520935.911720:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351635b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993146752:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.911729:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.911741:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.911742:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44439 00000004:00080000:3.0:1783520935.911746:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943634571f80/2, rc 0, transno 4294975941 00000100:00100000:3.0:1783520935.911772:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943634571f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993144448:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.911817:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e3c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.911845:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a81880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993146880:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.911895:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.911956:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993146880 00000100:00100000:2.0:1783520935.911965:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.911965:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993146880 00000100:00100000:2.0:1783520935.912000:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002413696 00000100:00100000:3.0:1783520935.912002:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e470380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993145728:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.912005:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44448 00000100:00100000:2.0:1783520935.912007:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002413696 00000100:00100000:1.0:1783520935.912017:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a81500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+375:3628:x1870152993146880:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.912032:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025b8f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x68b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.912032:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e86247cd/0xd5401cbabc02086f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x68b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d60 expref: 631 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.912035:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1682 count 1 00010000:00010000:1.0:1783520935.912050:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025bc0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x692:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.912074:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36033 00010000:00010000:1.0:1783520935.912081:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000059248741/0xd5401cbabc020899 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x692:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dc2 expref: 631 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.912085:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508059880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+631:2389:x1870153002413696:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.912099:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x692:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.912105:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.912105:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x692:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.912113:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025bc0 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x692:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.912114:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e86247cd/0xd5401cbabc02086f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x68b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d60 expref: 631 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.912139:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e86247cd/0xd5401cbabc02086f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x68b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d60 expref: 631 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.912154:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000059248741/0xd5401cbabc020899 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x692:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dc2 expref: 631 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.912156:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e86247cd/0xd5401cbabc02086f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x68b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d60 expref: 631 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.912182:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x68b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.912182:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000059248741/0xd5401cbabc020899 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x692:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dc2 expref: 630 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.912188:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x68b:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.912205:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516359880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993147008:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.912207:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025b8f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.912219:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025b8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x68b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.912241:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.912254:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e840 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.912260:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025b8f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.912271:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508059880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+630:2389:x1870153002413696:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 188us (310us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.912274:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025b8f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.912287:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025b8f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.912296:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36033 00000100:00100000:1.0:1783520935.912298:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.912307:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025b8f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x68b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.912316:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002413824 00000100:00100000:2.0:1783520935.912321:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002413824 00010000:00010000:3.0:1783520935.912325:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.912337:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000008908ae1/0xd5401cbabc020876 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x68c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d6e expref: 630 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.912391:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36034 00000100:00100000:2.0:1783520935.912400:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516358a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+630:2389:x1870153002413824:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.912409:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.912416:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000008908ae1/0xd5401cbabc020876 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x68c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d6e expref: 630 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.912432:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000008908ae1/0xd5401cbabc020876 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x68c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d6e expref: 630 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.912451:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x68c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.912455:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x68c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.912461:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025b81 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.912469:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025b81 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x68c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.912481:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.912521:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516358a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+630:2389:x1870153002413824:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 122us (229us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.912529:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36034 00010000:00010000:2.0:1783520935.912550:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.912554:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025b8f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x68b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.912567:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.912571:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025b8f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x68b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.912579:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025b8f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x68b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.912590:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025b8f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x68b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.912597:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025b8f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x68b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.912614:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.912616:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.912617:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.912642:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000008908ae1/0xd5401cbabc020876 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x68c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d6e expref: 629 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.912653:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993145472:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.912671:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025b81 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.912671:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.912683:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e473480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+376:3626:x1870152993145216:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4523us (4882us total) trans 4294975942 rc 0/0 00010000:00010000:2.0:1783520935.912685:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025b81 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.912701:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44442 00010000:00010000:2.0:1783520935.912702:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025b81 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.912708:0:35280:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002414080 00000100:00100000:1.0:1783520935.912714:0:35280:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002414080 00010000:00010000:2.0:1783520935.912721:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025b81 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.912731:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025b81 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x68c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.912736:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515777100/2, rc 0, transno 4294975942 00000100:00100000:0.0:1783520935.912750:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515777100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993145216:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.912751:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.912772:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520935.912772:0:35280:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98413 00010000:00010000:2.0:1783520935.912779:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025b81 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x68c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.912787:0:35280:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351635a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_011:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+33:251805:x1870153002414080:12345-192.168.204.33@tcp:36:dd.0 00010000:00010000:2.0:1783520935.912797:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.912802:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025b81 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x68c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.912810:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025b81 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x68c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.912827:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025b81 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x68c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.912833:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a83b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993147136:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.912847:0:35280:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.912850:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025b81 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x68c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.912871:0:35280:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff94351cbea000/0xd5401cbabc025b96 lrc: 0/0,0 mode: --/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 8 type: IBT gid 0 flags: 0x44a01000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f34ff expref: 33 pid: 6495 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.912874:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.912878:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.912884:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.912893:0:35280:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x72a:0x0].0x0 (ffff943515473e00) 00000100:00100000:0.0:1783520935.912895:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.912902:0:35280:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.912954:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993147136 00000100:00100000:0.0:1783520935.912960:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993147136 00000100:00100000:0.0:1783520935.912996:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44449 00000100:00100000:0.0:1783520935.913009:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436342b8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+377:3625:x1870152993147136:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.913018:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e473b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+377:3626:x1870152993145344:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5073us (5286us total) trans 4294975943 rc 0/0 00000100:00100000:2.0:1783520935.913032:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44441 00002000:00080000:0.0:1783520935.913035:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1683 count 1 00010000:00010000:0.0:1783520935.913057:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025bc7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x693:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.913072:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025bce lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x690:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.913077:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005712ee75/0xd5401cbabc0208a7 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x693:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dd0 expref: 628 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.913081:0:35280:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbac00/0xd5401cbabc025bd5 lrc: 3/0,1 mode: --/PW res: [0x200000402:0x72a:0x0].0x0 bits 0x0/0x0 rrc: 7 type: IBT flags: 0x40000000000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.913117:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e007b80/2, rc 0, transno 4294975943 00010000:00010000:1.0:1783520935.913118:0:35280:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbac00/0xd5401cbabc025bd5 lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 7 type: IBT flags: 0x50210000000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.913120:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x693:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.913124:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000fb49c6eb/0xd5401cbabc020892 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x690:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0da6 expref: 628 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.913141:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x693:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.913150:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e007b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993145344:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.913164:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025bc7 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x693:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.913178:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x690:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.913197:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x690:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.913200:0:35280:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbac00/0xd5401cbabc025bd5 lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 7 type: IBT flags: 0x40210000000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.913209:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025bce lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x690:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.913246:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000fb49c6eb/0xd5401cbabc020892 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x690:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0da6 expref: 628 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.913251:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:2.0:1783520935.913261:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000fb49c6eb/0xd5401cbabc020892 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x690:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0da6 expref: 628 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.913275:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9436342b9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993147264:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.913282:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350805ad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993147392:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.913327:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000001:00080000:1.0:1783520935.913330:0:35280:0:(tgt_lastrcvd.c:2117:tgt_txn_stop_cb()) More than one transaction 8589957903 00000004:00010000:1.0:1783520935.913412:0:35280:0:(mdt_handler.c:4478:mdt_save_lock()) ### save lock request 000000000fb92d9e reply state 00000000cfdaef14 transno 8589957904 ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbac00/0xd5401cbabc025bd5 lrc: 3/0,1 mode: PW/PW res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 7 type: IBT flags: 0x40210000000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.913427:0:35280:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbac00/0xd5401cbabc025bd5 lrc: 3/0,1 mode: TXN/TXN res: [0x200000402:0x72a:0x0].0x0 bits 0x2/0x0 rrc: 7 type: IBT flags: 0x50210000000000 pid: 35280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.913438:0:35280:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x72a:0x0].0x0 (ffff943515473e00) 00010000:00010000:1.0:1783520935.913441:0:35280:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00000100:00100000:1.0:1783520935.913444:0:35280:0:(service.c:179:ptlrpc_save_lock()) nlocks 0 00000100:00100000:2.0:1783520935.913455:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676d80 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.913457:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993147264 00000100:00100000:0.0:1783520935.913465:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993147264 00000100:00100000:0.0:1783520935.913523:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44450 00000100:00100000:0.0:1783520935.913545:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436342b8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+377:3625:x1870152993147264:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.913547:0:35280:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351635a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_011:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+35:251805:x1870153002414080:12345-192.168.204.33@tcp:36:dd.0 Request processed in 751us (864us total) trans 8589957904 rc 0/0 00000100:00100000:1.0:1783520935.913570:0:35280:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98413 00000020:00080000:1.0:1783520935.913576:0:35280:0:(tgt_main.c:784:tgt_ses_key_exit()) total 2 transactions per RPC 00002000:00080000:0.0:1783520935.913576:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1684 count 1 00010000:00010000:0.0:1783520935.913600:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025bdc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x694:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.913619:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002674e88d/0xd5401cbabc0208ae lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x694:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dde expref: 628 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.913632:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x694:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.913636:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x694:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.913640:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025bdc lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x694:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.913659:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351635aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993145856:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.913665:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e471f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993146240:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.913674:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002674e88d/0xd5401cbabc0208ae lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x694:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dde expref: 628 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.913685:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025ba4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x68d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.913690:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002674e88d/0xd5401cbabc0208ae lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x694:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dde expref: 628 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.913695:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025bab lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x68e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.913728:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.913754:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9436342b9500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993147520:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.913775:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351635b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993146752:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.913779:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.913799:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025bb9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x691:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.913817:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002414336 00000100:00100000:2.0:1783520935.913824:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002414336 00000100:00100000:0.0:1783520935.913843:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470900 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.913847:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002414592 00010000:00010000:2.0:1783520935.913858:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f48ba41a/0xd5401cbabc02088b lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x68f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d98 expref: 629 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.913858:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002414592 00010000:00010000:1.0:1783520935.913886:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000346e8ac0/0xd5401cbabc0208a0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x691:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0db4 expref: 630 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.913910:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36035 00000100:00100000:0.0:1783520935.913915:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a81c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993146368:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.913944:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94361b4a6050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+630:2388:x1870153002414336:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.913944:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36036 00010000:00010000:0.0:1783520935.913954:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025bb2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x68f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.913961:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.913963:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9ea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+630:2388:x1870153002414592:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.913981:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f48ba41a/0xd5401cbabc02088b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x68f:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d98 expref: 630 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.913990:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.914005:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000346e8ac0/0xd5401cbabc0208a0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x691:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0db4 expref: 630 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.914011:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f48ba41a/0xd5401cbabc02088b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x68f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d98 expref: 630 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.914025:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000346e8ac0/0xd5401cbabc0208a0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x691:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0db4 expref: 630 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.914037:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f48ba41a/0xd5401cbabc02088b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x68f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d98 expref: 630 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.914041:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005712ee75/0xd5401cbabc0208a7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x693:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dd0 expref: 630 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.914049:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000346e8ac0/0xd5401cbabc0208a0 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x691:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0db4 expref: 630 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.914092:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005712ee75/0xd5401cbabc0208a7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x693:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dd0 expref: 629 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.914097:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x68f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.914106:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x691:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.914119:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x68f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.914135:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x691:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.914148:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025bb2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.914160:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025bb9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x691:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.914183:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9436342ba300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993147648:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.914208:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025bb2 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x68f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.914208:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025bb9 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x691:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.914220:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470b40 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.914250:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.914265:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025bb2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.914277:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025bb2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.914285:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94361b4a6050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+628:2388:x1870153002414336:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 352us (568us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.914315:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36035 00010000:00010000:0.0:1783520935.914320:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025bb2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.914329:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025bb2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x68f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.914337:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.914347:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.914398:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.914403:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025bb2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x68f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.914414:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9ea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+627:2388:x1870153002414592:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 450us (629us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.914418:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.914425:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025bb2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x68f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.914461:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.914463:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36036 00010000:00010000:0.0:1783520935.914471:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025bb2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x68f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.914490:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002414208 00000100:00100000:1.0:1783520935.914507:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002414208 00010000:00010000:0.0:1783520935.914509:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025bb2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x68f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.914521:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025bb2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x68f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.914529:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a2630d1b/0xd5401cbabc02286d lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x68e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f5d expref: 627 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.914548:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.914554:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.914560:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.914567:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36037 00000100:00100000:1.0:1783520935.914578:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9f800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+627:2387:x1870153002414208:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.914589:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.914595:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a2630d1b/0xd5401cbabc02286d lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x68e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f5d expref: 627 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.914607:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a2630d1b/0xd5401cbabc02286d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x68e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f5d expref: 627 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.914625:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a2630d1b/0xd5401cbabc02286d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x68e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f5d expref: 627 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.914633:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516359880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993147008:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.914641:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x68e:0x0].0x0 since=0s v=(1 0) 00000100:00100000:2.0:1783520935.914642:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.914646:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x68e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.914653:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025bab lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.914663:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025bab lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x68e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.914664:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025bc0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x692:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.914715:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.914725:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a83100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+378:3625:x1870152993146112:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4680us (4909us total) trans 4294975944 rc 0/0 00000100:00100000:0.0:1783520935.914737:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44445 00000100:00100000:1.0:1783520935.914748:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9f800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+626:2387:x1870153002414208:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 170us (297us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.914756:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.914765:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36037 00000100:00100000:2.0:1783520935.914797:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002414464 00000100:00100000:1.0:1783520935.914798:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002414720 00000100:00100000:2.0:1783520935.914803:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002414464 00000100:00100000:1.0:1783520935.914805:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002414720 00000004:00080000:0.0:1783520935.914805:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943509a83800/2, rc 0, transno 4294975944 00000100:00100000:0.0:1783520935.914824:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a83800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993146112:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.914828:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000acb12ba4/0xd5401cbabc02087d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x68d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d7c expref: 627 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.914833:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000059248741/0xd5401cbabc020899 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x692:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dc2 expref: 627 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.914887:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36038 00000100:00100000:1.0:1783520935.914898:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36039 00000100:00100000:2.0:1783520935.914901:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e006d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+627:2387:x1870153002414464:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.914917:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.914919:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508059f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+627:2387:x1870153002414720:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.914928:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9436342bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993147776:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.914929:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000acb12ba4/0xd5401cbabc02087d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x68d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d7c expref: 627 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.914935:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.914950:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000059248741/0xd5401cbabc020899 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x692:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dc2 expref: 627 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.914960:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000acb12ba4/0xd5401cbabc02087d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x68d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d7c expref: 627 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.914970:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000059248741/0xd5401cbabc020899 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x692:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dc2 expref: 627 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.914983:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000acb12ba4/0xd5401cbabc02087d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x68d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0d7c expref: 627 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.914996:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000059248741/0xd5401cbabc020899 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x692:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dc2 expref: 627 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.915000:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.915035:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x692:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.915037:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x68d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.915041:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x692:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.915043:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x68d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.915050:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025bc0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x692:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.915051:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025ba4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.915074:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025bc0 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x692:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.915075:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025ba4 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x68d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.915084:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993147776 00000100:00100000:3.0:1783520935.915090:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993147776 00010000:00010000:1.0:1783520935.915117:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.915118:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.915124:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44451 00000100:00100000:3.0:1783520935.915131:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436342b9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+378:3627:x1870152993147776:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520935.915148:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1685 count 1 00000100:00100000:1.0:1783520935.915153:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508059f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+625:2387:x1870153002414720:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 243us (401us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.915154:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e006d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+625:2387:x1870153002414464:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 256us (513us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.915165:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025be3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x695:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.915166:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36039 00000100:00100000:2.0:1783520935.915175:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36038 00010000:00010000:3.0:1783520935.915180:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d9d94c6a/0xd5401cbabc0208b5 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x695:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dec expref: 624 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.915186:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x695:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.915188:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025bab lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.915189:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x695:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.915197:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025be3 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x695:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.915200:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025bab lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.915211:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025bab lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.915221:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d9d94c6a/0xd5401cbabc0208b5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x695:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dec expref: 623 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.915225:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025bab lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x68e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.915244:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d9d94c6a/0xd5401cbabc0208b5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x695:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dec expref: 623 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.915251:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520935.915259:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e471f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993147904:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.915267:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.915274:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025bab lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x68e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.915283:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.915289:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025bab lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x68e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.915298:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025bab lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x68e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.915312:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025bab lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x68e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.915325:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025bab lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x68e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.915343:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8240 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.915348:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.915353:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.915376:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025bc0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x692:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.915383:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.915390:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025bc0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x692:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.915397:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e471c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+379:3627:x1870152993145984:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5394us (5635us total) trans 4294975945 rc 0/0 00010000:00010000:3.0:1783520935.915409:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025bc0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x692:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.915409:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44444 00010000:00010000:3.0:1783520935.915419:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025bc0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x692:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.915434:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025ba4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.915439:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.915444:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.915454:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025bc0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x692:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.915454:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025ba4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.915471:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025ba4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x68d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.915474:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbae00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.915479:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025ba4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x68d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.915481:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025bc0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x692:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.915505:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.915506:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025bc0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x692:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.915521:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025bc0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x692:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.915528:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025bc0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x692:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.915550:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.915554:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025ba4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x68d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.915562:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.915566:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025ba4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x68d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.915575:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e472300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+379:3627:x1870152993145600:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6114us (7092us total) trans 4294975947 rc 0/0 00010000:00010000:2.0:1783520935.915578:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025ba4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x68d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.915588:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44443 00010000:00010000:2.0:1783520935.915598:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025ba4 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x68d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.915607:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025ba4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x68d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.915616:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025bb9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x691:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.915623:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a81500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+379:3628:x1870152993146880:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3607us (3729us total) trans 4294975946 rc 0/0 00010000:00010000:1.0:1783520935.915625:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025bb9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x691:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.915636:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44448 00010000:00010000:1.0:1783520935.915639:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025bb9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x691:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.915644:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.915645:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025bb9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x691:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.915649:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.915653:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.915664:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.915667:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.915674:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025bb9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x691:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.915682:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.915687:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025bb9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x691:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.915695:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025bb9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x691:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.915707:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025bb9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x691:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.915717:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c14dd0e7/0xd5401cbabc025bb9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x691:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.915734:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a83480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+379:3627:x1870152993146624:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4188us (4306us total) trans 4294975948 rc 0/0 00010000:00010000:2.0:1783520935.915735:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.915738:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.915741:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.915746:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44447 00010000:00010000:0.0:1783520935.915826:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520935.915860:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943509a81f80/2, rc 0, transno 4294975948 00000100:00100000:0.0:1783520935.915870:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a81f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993146624:0@lo:6:osp-syn-1-0.0 00000004:00080000:0.0:1783520935.915890:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943509a80000/2, rc 0, transno 4294975945 00000100:00100000:0.0:1783520935.915894:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a80000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993145984:0@lo:6:osp-syn-1-0.0 00000004:00080000:0.0:1783520935.915905:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e007480/2, rc 0, transno 4294975947 00000100:00100000:0.0:1783520935.915908:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e007480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993145600:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.915973:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1783520935.915991:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9436342b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993148032:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.916005:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9436342b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993148288:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.916045:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:0.0:1783520935.916074:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943509a81880/2, rc 0, transno 4294975946 00000100:00100000:0.0:1783520935.916089:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a81880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993146880:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.916127:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993148032 00000100:00100000:1.0:1783520935.916135:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993148032 00000100:00100000:0.0:1783520935.916161:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.916169:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9436342bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993148416:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.916175:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993148288 00000100:00100000:1.0:1783520935.916180:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993148288 00000100:00100000:0.0:1783520935.916201:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1783520935.916214:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9436342b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993148160:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.916225:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44452 00000100:00100000:3.0:1783520935.916227:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.916238:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e470380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+380:3627:x1870152993148032:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.916255:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1686 count 1 00010000:00010000:1.0:1783520935.916269:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025bea lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x696:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.916282:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003295d90d/0xd5401cbabc022ac7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x696:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f31b7 expref: 623 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.916289:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x696:0x0].0x0 since=36s v=(0 0) 00010000:00010000:1.0:1783520935.916293:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x696:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.916297:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025bea lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x696:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.916322:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003295d90d/0xd5401cbabc022ac7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x696:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31b7 expref: 623 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.916336:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003295d90d/0xd5401cbabc022ac7 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x696:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31b7 expref: 623 pid: 9151 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.916352:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351635b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993148544:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.916355:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.916433:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993148416 00000100:00100000:3.0:1783520935.916438:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993148416 00000100:00100000:3.0:1783520935.916464:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993148160 00000100:00100000:3.0:1783520935.916466:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993148160 00000100:00100000:3.0:1783520935.916507:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44453 00000100:00100000:1.0:1783520935.916512:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.916527:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436342baa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+382:3626:x1870152993148288:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.916527:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44454 00000100:00100000:1.0:1783520935.916537:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44455 00000100:00100000:2.0:1783520935.916542:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e473800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+382:3627:x1870152993148416:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520935.916544:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1688 count 1 00000100:00100000:1.0:1783520935.916551:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436342bad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+382:3626:x1870152993148160:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.916562:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025bf1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x698:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.916562:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1689 count 1 00002000:00080000:1.0:1783520935.916568:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1687 count 1 00010000:00010000:2.0:1783520935.916580:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025bf8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x699:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.916581:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ca2f8956/0xd5401cbabc0208ca lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x698:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e16 expref: 623 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.916586:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025bff lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x697:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.916603:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x698:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.916610:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x698:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.916618:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025bf1 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x698:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.916620:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c1773f89/0xd5401cbabc0208d1 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x699:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e24 expref: 623 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.916622:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f2a57608/0xd5401cbabc0208c3 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x697:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e08 expref: 623 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.916645:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x699:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.916646:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x697:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.916655:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x699:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.916655:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x697:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.916657:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ca2f8956/0xd5401cbabc0208ca lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x698:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e16 expref: 623 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.916667:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025bf8 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x699:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.916667:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025bff lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x697:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.916673:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ca2f8956/0xd5401cbabc0208ca lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x698:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e16 expref: 623 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.916699:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d320700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993148672:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.916708:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f2a57608/0xd5401cbabc0208c3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x697:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e08 expref: 623 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.916713:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c1773f89/0xd5401cbabc0208d1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x699:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e24 expref: 623 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.916725:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8180 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.916733:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f2a57608/0xd5401cbabc0208c3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x697:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e08 expref: 623 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.916746:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c1773f89/0xd5401cbabc0208d1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x699:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e24 expref: 623 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.916747:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351635aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993148800:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.916769:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350805b100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993148928:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.916780:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3eb40 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.916805:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676240 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.918926:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350805ad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993147392:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.918966:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025bce lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x690:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.919908:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.919945:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.919984:0:24275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002414976 00000100:00100000:0.0:1783520935.920000:0:24275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002414976 00000100:00100000:2.0:1783520935.920001:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002414848 00000100:00100000:2.0:1783520935.920022:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002414848 00010000:00010000:2.0:1783520935.920070:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000fb49c6eb/0xd5401cbabc020892 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x690:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0da6 expref: 625 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00010000:0.0:1783520935.920071:0:24275:0:(ofd_dev.c:2825:ofd_punch_hpreq_check()) lustre-OST0001: refresh locks: 11811161089/2295 (196608->18446744073709551615) 00002000:00100000:0.0:1783520935.920085:0:24275:0:(ofd_dev.c:2565:ofd_prolong_extent_locks()) Prolong locks for req ffff943516359c00 with x1870153002414976 ext(196608->18446744073709551615) 00000100:00100000:3.0:1783520935.920086:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e471f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993147904:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.920090:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00002000:00010000:0.0:1783520935.920112:0:24275:0:(ofd_dev.c:2830:ofd_punch_hpreq_check()) lustre-OST0001: refreshed 0 locks timeout for req ffff943516359c00. 00010000:00010000:3.0:1783520935.920126:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025be3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x695:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.920133:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002415104 00000100:00100000:1.0:1783520935.920138:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002415104 00000100:00100000:2.0:1783520935.920142:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36040 00010000:00010000:1.0:1783520935.920166:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002674e88d/0xd5401cbabc0208ae lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x694:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dde expref: 626 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.920168:0:24275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 34737 00000100:00100000:2.0:1783520935.920170:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516359180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+626:2390:x1870153002414848:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.920195:0:24275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516359c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_io00_010:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+626:2388:x1870153002414976:12345-192.168.204.33@tcp:10:dd.0 00010000:00010000:2.0:1783520935.920199:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.920208:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36041 00010000:00010000:2.0:1783520935.920213:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000fb49c6eb/0xd5401cbabc020892 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x690:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0da6 expref: 626 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.920224:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516358000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+626:2387:x1870153002415104:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.920231:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000fb49c6eb/0xd5401cbabc020892 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x690:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0da6 expref: 626 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.920241:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.920248:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002674e88d/0xd5401cbabc0208ae lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x694:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dde expref: 626 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.920261:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000fb49c6eb/0xd5401cbabc020892 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x690:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0da6 expref: 626 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.920263:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002674e88d/0xd5401cbabc0208ae lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x694:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dde expref: 626 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.920289:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x690:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.920293:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x694:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.920295:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x690:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.920302:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x694:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.920306:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025bce lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x690:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.920309:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025bdc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x694:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.920330:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025bce lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x690:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.920336:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025bdc lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x694:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.920395:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.920398:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.920403:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025bce lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x690:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.920415:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025bce lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x690:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.920428:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025bce lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x690:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.920439:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025bce lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x690:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.920446:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516358000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+625:2387:x1870153002415104:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 228us (365us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.920450:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516359180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+626:2390:x1870153002414848:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 292us (500us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.920461:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36041 00010000:00010000:3.0:1783520935.920474:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520935.920475:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36040 00010000:00010000:2.0:1783520935.920518:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00002000:00010000:0.0:1783520935.920521:0:24275:0:(ofd_lvb.c:306:ofd_lvbo_update()) res: [0x2c0000401:0x8f7:0x0] updating lvb size from disk: 65536 -> 196608 00010000:00010000:2.0:1783520935.920525:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025bce lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x690:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00010000:0.0:1783520935.920528:0:24275:0:(ofd_lvb.c:313:ofd_lvbo_update()) res: [0x2c0000401:0x8f7:0x0] updating lvb mtime from disk: 1783520936 -> 1783520936 00000100:00100000:1.0:1783520935.920530:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d320700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993148672:192.168.204.33@tcp:104: 00002000:00010000:0.0:1783520935.920543:0:24275:0:(ofd_lvb.c:319:ofd_lvbo_update()) res: [0x2c0000401:0x8f7:0x0] updating lvb atime from disk: 0 -> 0 00010000:00010000:2.0:1783520935.920559:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184000 already unused, calling callback (ffffffffc0bf9e90) 00002000:00010000:0.0:1783520935.920559:0:24275:0:(ofd_lvb.c:325:ofd_lvbo_update()) res: [0x2c0000401:0x8f7:0x0] updating lvb ctime from disk: 1783520936 -> 1783520936 00010000:00010000:1.0:1783520935.920560:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025bf1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x698:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00010000:0.0:1783520935.920572:0:24275:0:(ofd_lvb.c:331:ofd_lvbo_update()) res: [0x2c0000401:0x8f7:0x0] updating lvb blocks from disk: 0 -> 0 00010000:00010000:2.0:1783520935.920583:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025bce lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x690:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.920599:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025bce lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x690:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.920614:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025bce lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x690:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.920623:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025bce lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x690:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.920640:0:24275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516359c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_io00_010:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+624:2388:x1870153002414976:12345-192.168.204.33@tcp:10:dd.0 Request processed in 450us (729us total) trans 4294975949 rc 0/0 00010000:00010000:2.0:1783520935.920648:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.920651:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.920654:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.920657:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.920664:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e470000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+383:3626:x1870152993146496:12345-0@lo:6:osp-syn-1-0.0 Request processed in 9328us (9450us total) trans 4294975950 rc 0/0 00000100:00100000:0.0:1783520935.920670:0:24275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 34737 00002000:00010000:0.0:1783520935.920683:0:24275:0:(ofd_dev.c:2825:ofd_punch_hpreq_check()) lustre-OST0001: refresh locks: 11811161089/2295 (196608->18446744073709551615) 00000100:00100000:3.0:1783520935.920690:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44446 00002000:00100000:0.0:1783520935.920691:0:24275:0:(ofd_dev.c:2565:ofd_prolong_extent_locks()) Prolong locks for req ffff943516359c00 with x1870153002414976 ext(196608->18446744073709551615) 00002000:00010000:0.0:1783520935.920704:0:24275:0:(ofd_dev.c:2830:ofd_punch_hpreq_check()) lustre-OST0001: refreshed 0 locks timeout for req ffff943516359c00. 00000100:00100000:2.0:1783520935.920776:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002415232 00000100:00100000:2.0:1783520935.920783:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002415232 00000004:00080000:0.0:1783520935.920796:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943509a82680/2, rc 0, transno 4294975950 00000100:00100000:1.0:1783520935.920798:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.920809:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005712ee75/0xd5401cbabc0208a7 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x693:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dd0 expref: 624 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.920824:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a82680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993146496:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.920845:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002415360 00000100:00100000:2.0:1783520935.920852:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002415360 00000100:00100000:1.0:1783520935.920857:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36042 00010000:00010000:2.0:1783520935.920869:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d9d94c6a/0xd5401cbabc0208b5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x695:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dec expref: 625 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.920870:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f5850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+625:2387:x1870153002415232:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.920887:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.920895:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005712ee75/0xd5401cbabc0208a7 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x693:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dd0 expref: 625 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.920901:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36043 00010000:00010000:1.0:1783520935.920922:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005712ee75/0xd5401cbabc0208a7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x693:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dd0 expref: 625 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.920923:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516359f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+625:2390:x1870153002415360:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.920939:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.920945:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x693:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.920946:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d9d94c6a/0xd5401cbabc0208b5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x695:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dec expref: 625 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.920953:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x693:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.920953:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a81c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993149056:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.920976:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025bc7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x693:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.920980:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d9d94c6a/0xd5401cbabc0208b5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x695:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dec expref: 625 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.921009:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025bc7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x693:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.921016:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d9d94c6a/0xd5401cbabc0208b5 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x695:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dec expref: 625 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.921032:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.921038:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.921039:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x695:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.921047:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x695:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.921052:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025be3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x695:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.921062:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025be3 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x695:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.921076:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f5850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+624:2387:x1870153002415232:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 207us (430us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.921082:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.921086:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36042 00000100:00100000:2.0:1783520935.921111:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516359f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+623:2390:x1870153002415360:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 191us (313us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.921120:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36043 00010000:00010000:3.0:1783520935.921136:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025be3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x695:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.921144:0:35282:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002674e88d/0xd5401cbabc0208ae lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x694:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dde expref: 622 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.921160:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025be3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x695:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.921180:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025be3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x695:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.921186:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9436342b9500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993147520:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.921206:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025be3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x695:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.921230:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.921231:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025bdc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x694:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.921247:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005712ee75/0xd5401cbabc0208a7 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x693:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0dd0 expref: 621 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.921252:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.921252:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025bdc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x694:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.921266:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025be3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x695:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.921282:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9436342ba300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993147648:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.921302:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025bdc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x694:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.921303:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.921308:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025be3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x695:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.921319:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025bc7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x693:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.921351:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025be3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x695:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.921354:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025bc7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x693:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.921354:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025bdc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x694:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.921416:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025bc7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x693:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.921419:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025be3 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x695:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.921423:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436342b9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+383:3627:x1870152993147776:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6288us (6434us total) trans 4294975951 rc 0/0 00010000:00010000:0.0:1783520935.921469:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025bdc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x694:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.921475:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025bc7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x693:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.921526:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025be3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x695:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.921527:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025bc7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x693:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.921551:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44451 00010000:00010000:0.0:1783520935.921561:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520935.921568:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520935.921572:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025bdc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x694:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.921573:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.921582:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.921587:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025bdc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x694:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.921594:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.921596:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025bdc lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x694:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.921600:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9436342bb480/2, rc 0, transno 4294975951 00010000:00010000:0.0:1783520935.921603:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025bc7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x693:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.921622:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9436342bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993147776:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.921624:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025bdc lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x694:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.921629:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.921639:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025bc7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x693:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.921640:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025bdc lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x694:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.921649:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993149056 00000100:00100000:3.0:1783520935.921657:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993149056 00000100:00100000:1.0:1783520935.921677:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436342b8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+384:3625:x1870152993147136:12345-0@lo:6:osp-syn-1-0.0 Request processed in 8666us (8778us total) trans 4294975952 rc 0/0 00000100:00100000:3.0:1783520935.921690:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44456 00010000:00010000:0.0:1783520935.921690:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025bc7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x693:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.921700:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.921705:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44449 00000100:00100000:3.0:1783520935.921708:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d28a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+384:3628:x1870152993149056:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.921708:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.921713:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.921728:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.921734:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025bc7 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x693:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520935.921737:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1690 count 1 00010000:00010000:0.0:1783520935.921752:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025bc7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x693:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.921761:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025c06 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x69a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.921761:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943509a83b80/2, rc 0, transno 4294975952 00000100:00100000:1.0:1783520935.921777:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a83b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993147136:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.921782:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.921784:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c23b3eb1/0xd5401cbabc0208d8 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x69a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e32 expref: 620 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.921789:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.921798:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.921799:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x69a:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.921805:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x69a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.921811:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025c06 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x69a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.921815:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350805b100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993148928:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.921823:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351635b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993148544:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.921843:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025bf8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x699:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.921851:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025bea lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x696:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.921854:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c23b3eb1/0xd5401cbabc0208d8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x69a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e32 expref: 620 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.921883:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c23b3eb1/0xd5401cbabc0208d8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x69a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e32 expref: 620 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.921897:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993149440:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.921904:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d321f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993149312:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.921918:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351635aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993148800:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.921928:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.921937:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025bff lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x697:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.921959:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002415616 00000100:00100000:2.0:1783520935.921966:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002415616 00000100:00100000:0.0:1783520935.921975:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.921990:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003295d90d/0xd5401cbabc022ac7 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x696:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31b7 expref: 621 pid: 9151 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.921993:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8780 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.922021:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1783520935.922023:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993149440 00000100:00100000:3.0:1783520935.922030:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993149440 00000100:00100000:2.0:1783520935.922030:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36044 00000100:00100000:0.0:1783520935.922038:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993149184:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.922044:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350805b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+621:2388:x1870153002415616:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.922054:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.922060:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003295d90d/0xd5401cbabc022ac7 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x696:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31b7 expref: 621 pid: 9151 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.922072:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44457 00010000:00010000:2.0:1783520935.922077:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003295d90d/0xd5401cbabc022ac7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x696:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31b7 expref: 621 pid: 9151 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.922084:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436342b8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+385:3625:x1870152993147264:12345-0@lo:6:osp-syn-1-0.0 Request processed in 8549us (8755us total) trans 4294975953 rc 0/0 00000100:00100000:3.0:1783520935.922086:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d28000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+385:3626:x1870152993149440:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.922099:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.922114:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44450 00002000:00080000:3.0:1783520935.922124:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1692 count 1 00010000:00010000:2.0:1783520935.922125:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003295d90d/0xd5401cbabc022ac7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x696:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31b7 expref: 621 pid: 9151 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.922150:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025c0d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x69c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.922150:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x696:0x0].0x0 since=0s v=(1 0) 00000100:00100000:1.0:1783520935.922150:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993149184 00000100:00100000:1.0:1783520935.922156:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993149184 00010000:00010000:2.0:1783520935.922158:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x696:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.922167:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025bea lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x696:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.922175:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9436342b9880/2, rc 0, transno 4294975953 00010000:00010000:3.0:1783520935.922176:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000805870f0/0xd5401cbabc0208df lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x69c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e4e expref: 620 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.922189:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44458 00000100:00100000:0.0:1783520935.922194:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9436342b9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993147264:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.922209:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d2a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+385:3626:x1870152993149184:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.922213:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025bea lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x696:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.922230:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1691 count 1 00010000:00010000:3.0:1783520935.922235:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x69c:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.922241:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x69c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.922247:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025c0d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x69c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.922251:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025c14 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x69b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.922254:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.922271:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a0fb4ad2/0xd5401cbabc0208e6 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x69b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e40 expref: 620 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.922274:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993149568:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.922286:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000805870f0/0xd5401cbabc0208df lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x69c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e4e expref: 620 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.922288:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x69b:0x0].0x0 since=37s v=(0 0) 00000100:00100000:2.0:1783520935.922290:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350805b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+620:2388:x1870153002415616:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 242us (357us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.922300:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x69b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.922314:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025c14 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x69b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.922319:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36044 00010000:00010000:3.0:1783520935.922327:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000805870f0/0xd5401cbabc0208df lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x69c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e4e expref: 620 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.922346:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025bea lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x696:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.922347:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d320000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993149696:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.922353:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.922400:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025bea lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x696:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.922412:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025bea lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x696:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.922415:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.922424:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025bea lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x696:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.922444:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.922451:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.922459:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025bea lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x696:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.922467:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba600 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:3.0:1783520935.922471:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8a80 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.922472:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025bea lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x696:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.922484:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025bea lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x696:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.922511:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993149568 00010000:00010000:2.0:1783520935.922514:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025bea lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x696:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.922519:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993149568 00010000:00010000:2.0:1783520935.922529:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025bea lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x696:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.922566:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e470380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+387:3627:x1870152993148032:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6327us (6518us total) trans 4294975954 rc 0/0 00010000:00010000:2.0:1783520935.922572:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.922578:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.922578:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44459 00010000:00010000:2.0:1783520935.922584:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.922589:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44452 00000004:00080000:3.0:1783520935.922592:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9436342b9f80/2, rc 0, transno 4294975954 00000100:00100000:0.0:1783520935.922597:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d29c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+387:3627:x1870152993149568:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.922602:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002415872 00000100:00100000:2.0:1783520935.922609:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002415872 00000100:00100000:3.0:1783520935.922615:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9436342b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993148032:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.922634:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f2a57608/0xd5401cbabc0208c3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x697:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e08 expref: 620 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.922651:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a0fb4ad2/0xd5401cbabc0208e6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x69b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e40 expref: 620 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.922660:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a0fb4ad2/0xd5401cbabc0208e6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x69b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e40 expref: 620 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.922668:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36045 00000100:00100000:1.0:1783520935.922674:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351635aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993149952:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.922684:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508059880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+620:2388:x1870153002415872:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.922690:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993149824:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.922699:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.922707:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f2a57608/0xd5401cbabc0208c3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x697:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e08 expref: 620 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.922709:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.922722:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f2a57608/0xd5401cbabc0208c3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x697:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e08 expref: 620 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.922736:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.922746:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f2a57608/0xd5401cbabc0208c3 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x697:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e08 expref: 620 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.922764:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x697:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.922768:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x697:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.922772:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025bff lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x697:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.922773:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993149824 00000100:00100000:1.0:1783520935.922780:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993149824 00010000:00010000:2.0:1783520935.922789:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025bff lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x697:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.922804:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1693 count 1 00010000:00010000:2.0:1783520935.922811:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.922831:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44460 00010000:00010000:0.0:1783520935.922831:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025c1b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x69d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.922844:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508059880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+619:2388:x1870153002415872:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 165us (431us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.922856:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d29180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+387:3628:x1870152993149824:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.922879:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36045 00002000:00080000:1.0:1783520935.922886:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1694 count 1 00010000:00010000:0.0:1783520935.922887:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000085e8ed28/0xd5401cbabc0208ed lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x69d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e5c expref: 619 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.922902:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025bff lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x697:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.922902:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x69d:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.922907:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025c22 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x69e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.922924:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x69d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.922946:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025c1b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x69d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.922947:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025bff lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x697:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.922958:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001bfd0fc9/0xd5401cbabc0229b6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x69e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f30a6 expref: 618 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.922995:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025bff lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x697:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.922995:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x69e:0x0].0x0 since=36s v=(0 0) 00010000:00010000:1.0:1783520935.923004:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x69e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.923010:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000085e8ed28/0xd5401cbabc0208ed lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x69d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e5c expref: 618 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.923021:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025bff lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x697:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.923021:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025c22 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x69e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.923040:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000085e8ed28/0xd5401cbabc0208ed lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x69d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e5c expref: 618 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.923064:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.923073:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d2bb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993150080:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.923083:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001bfd0fc9/0xd5401cbabc0229b6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x69e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30a6 expref: 618 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.923103:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001bfd0fc9/0xd5401cbabc0229b6 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x69e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30a6 expref: 618 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.923118:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351635b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993150208:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.923123:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470e40 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.923160:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e3c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.923184:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436342bad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+388:3626:x1870152993148160:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6636us (6833us total) trans 4294975955 rc 0/0 00000100:00100000:2.0:1783520935.923194:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44455 00000004:00080000:0.0:1783520935.923205:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9436342b8700/2, rc 0, transno 4294975955 00000100:00100000:0.0:1783520935.923221:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9436342b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993148160:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.923225:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.923234:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025bff lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x697:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.923244:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.923249:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025bff lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x697:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.923257:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025bff lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x697:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.923280:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:2.0:1783520935.923281:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025bff lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x697:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.923295:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025bff lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x697:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.923298:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993150336:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.923319:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.923350:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.923383:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002415488 00000100:00100000:3.0:1783520935.923389:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002415488 00010000:00010000:3.0:1783520935.923418:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ca2f8956/0xd5401cbabc0208ca lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x698:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e16 expref: 619 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.923418:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.923430:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.923438:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.923467:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36046 00000100:00100000:0.0:1783520935.923483:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993150336 00000100:00100000:2.0:1783520935.923484:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d321c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+619:2388:x1870153002415488:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.923489:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993150336 00010000:00010000:2.0:1783520935.923508:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.923516:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ca2f8956/0xd5401cbabc0208ca lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x698:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e16 expref: 619 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.923530:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ca2f8956/0xd5401cbabc0208ca lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x698:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e16 expref: 619 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.923544:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44461 00010000:00010000:2.0:1783520935.923552:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ca2f8956/0xd5401cbabc0208ca lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x698:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e16 expref: 619 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.923558:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d29f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+388:3626:x1870152993150336:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.923582:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x698:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.923586:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x698:0x0].0x0 since=0s v=(0 0) 00002000:00080000:0.0:1783520935.923587:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1695 count 1 00010000:00010000:2.0:1783520935.923592:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025bf1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x698:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.923604:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025bf1 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x698:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.923608:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025c29 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x69f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.923636:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000406c3132/0xd5401cbabc0208fb lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x69f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e78 expref: 618 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.923644:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.923652:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.923656:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025bf1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x698:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.923670:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x69f:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.923676:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025bf1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x698:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.923677:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x69f:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.923687:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d321c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+618:2388:x1870153002415488:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 205us (368us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.923696:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025c29 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x69f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.923724:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36046 00010000:00010000:1.0:1783520935.923725:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025bf1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x698:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.923741:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025bf1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x698:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.923749:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002415744 00000100:00100000:2.0:1783520935.923755:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002415744 00010000:00010000:0.0:1783520935.923760:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000406c3132/0xd5401cbabc0208fb lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x69f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e78 expref: 618 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.923763:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.923768:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.923778:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025bf1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x698:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.923787:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000406c3132/0xd5401cbabc0208fb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x69f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e78 expref: 618 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.923791:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.923799:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025bf1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x698:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.923825:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d2ad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993150464:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.923843:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025bf1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x698:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.923860:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025bf1 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x698:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.923868:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025bf1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x698:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.923889:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436342baa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+389:3626:x1870152993148288:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7360us (7725us total) trans 4294975956 rc 0/0 00010000:00010000:2.0:1783520935.923894:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.923900:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.923904:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44453 00010000:00010000:2.0:1783520935.923908:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.923914:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470900 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.923937:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c1773f89/0xd5401cbabc0208d1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x699:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e24 expref: 618 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.923964:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9436342b9c00/2, rc 0, transno 4294975956 00000100:00100000:2.0:1783520935.923973:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36047 00000100:00100000:0.0:1783520935.923981:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9436342b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993148288:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.923994:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d321880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+618:2388:x1870153002415744:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.924006:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.924012:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c1773f89/0xd5401cbabc0208d1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x699:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e24 expref: 618 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.924024:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c1773f89/0xd5401cbabc0208d1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x699:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e24 expref: 618 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.924056:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c1773f89/0xd5401cbabc0208d1 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x699:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e24 expref: 618 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.924062:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993150592:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.924080:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x699:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.924085:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x699:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.924090:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025bf8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x699:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.924101:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.924110:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025bf8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x699:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.924137:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.924137:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993150592 00000100:00100000:0.0:1783520935.924144:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993150592 00000100:00100000:2.0:1783520935.924176:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d321880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+617:2388:x1870153002415744:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 192us (538us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.924188:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36047 00000100:00100000:0.0:1783520935.924195:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44462 00000100:00100000:0.0:1783520935.924208:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d29880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+389:3626:x1870152993150592:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.924226:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1696 count 1 00010000:00010000:2.0:1783520935.924233:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025bf8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x699:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.924247:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025c30 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6a0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.924248:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025bf8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x699:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.924294:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.924303:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025bf8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x699:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.924306:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d8bd7620/0xd5401cbabc020902 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e86 expref: 616 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.924338:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025bf8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x699:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.924340:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a0:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.924353:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.924466:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.924479:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025c30 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.924510:0:25736:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002416128 00010000:00010000:0.0:1783520935.924510:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:3.0:1783520935.924520:0:25736:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002416128 00010000:00010000:0.0:1783520935.924521:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025bf8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x699:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.924534:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.924541:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025bf8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x699:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.924551:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025bf8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x699:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.924573:0:25736:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 9628 00010000:00010000:0.0:1783520935.924574:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025bf8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x699:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.924588:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025bf8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x699:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.924589:0:25736:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516358000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_rdpg00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+35:251805:x1870153002416128:12345-192.168.204.33@tcp:35:dd.0 00000100:00100000:2.0:1783520935.924600:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e473800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+390:3627:x1870152993148416:12345-0@lo:6:osp-syn-1-0.0 Request processed in 8057us (8375us total) trans 4294975957 rc 0/0 00000100:00100000:2.0:1783520935.924635:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44454 00010000:00010000:0.0:1783520935.924662:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.924667:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.924670:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:2.0:1783520935.924701:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9436342bb800/2, rc 0, transno 4294975957 00010000:00010000:0.0:1783520935.924708:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d8bd7620/0xd5401cbabc020902 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e86 expref: 616 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.924725:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d8bd7620/0xd5401cbabc020902 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e86 expref: 616 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.924732:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9436342bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993148416:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.924755:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d28380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993150720:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.924823:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1783520935.924836:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993150848:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.924876:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.924934:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993150848 00000100:00100000:3.0:1783520935.924939:0:25736:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516358000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_rdpg00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+36:251805:x1870153002416128:12345-192.168.204.33@tcp:35:dd.0 Request processed in 353us (665us total) trans 8589957905 rc 0/0 00000100:00100000:1.0:1783520935.924942:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993150848 00000100:00100000:3.0:1783520935.924957:0:25736:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 9628 00000100:00100000:0.0:1783520935.924962:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470b40 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.924988:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44463 00000100:00100000:1.0:1783520935.925002:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436342ba300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+390:3628:x1870152993150848:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.925021:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1697 count 1 00010000:00010000:1.0:1783520935.925082:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025c37 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6a1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.925105:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000fe5852fe/0xd5401cbabc02091e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e94 expref: 616 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.925113:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a1:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.925117:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.925122:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025c37 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.925157:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000fe5852fe/0xd5401cbabc02091e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e94 expref: 616 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.925169:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000fe5852fe/0xd5401cbabc02091e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e94 expref: 616 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.925183:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516359180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993150976:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.925236:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.925516:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.926130:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d321f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993149312:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.926156:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025c06 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x69a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.926217:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351635aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993149952:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.926254:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025c14 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x69b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.926308:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d2bb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993150080:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.926329:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025c1b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x69d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.926649:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.926682:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.926765:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d320000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993149696:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.926784:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025c0d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x69c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.926853:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002416256 00000100:00100000:2.0:1783520935.926860:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002416256 00000100:00100000:1.0:1783520935.926873:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.926893:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c23b3eb1/0xd5401cbabc0208d8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x69a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e32 expref: 617 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.926907:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.926927:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002416384 00000100:00100000:2.0:1783520935.926933:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002416384 00000100:00100000:1.0:1783520935.926948:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002416512 00010000:00010000:2.0:1783520935.926955:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a0fb4ad2/0xd5401cbabc0208e6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x69b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e40 expref: 618 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.926957:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002416512 00000100:00100000:2.0:1783520935.926987:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002416640 00000100:00100000:2.0:1783520935.926995:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002416640 00010000:00010000:1.0:1783520935.926996:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000085e8ed28/0xd5401cbabc0208ed lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x69d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e5c expref: 619 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.927020:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000805870f0/0xd5401cbabc0208df lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x69c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e4e expref: 620 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.927034:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36048 00000100:00100000:1.0:1783520935.927052:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351635aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+620:2389:x1870153002416256:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.927061:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36049 00010000:00010000:1.0:1783520935.927069:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520935.927076:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f0050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+620:2389:x1870153002416384:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.927084:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c23b3eb1/0xd5401cbabc0208d8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x69a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e32 expref: 620 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.927094:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.927107:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a0fb4ad2/0xd5401cbabc0208e6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x69b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e40 expref: 620 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.927107:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c23b3eb1/0xd5401cbabc0208d8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x69a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e32 expref: 620 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.927150:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36050 00010000:00010000:2.0:1783520935.927165:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a0fb4ad2/0xd5401cbabc0208e6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x69b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e40 expref: 620 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.927169:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516359880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+620:2388:x1870153002416512:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.927175:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c23b3eb1/0xd5401cbabc0208d8 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x69a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e32 expref: 620 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.927190:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.927209:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x69a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.927215:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x69a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.927221:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025c06 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.927235:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a0fb4ad2/0xd5401cbabc0208e6 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x69b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e40 expref: 619 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.927236:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025c06 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x69a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.927252:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000085e8ed28/0xd5401cbabc0208ed lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x69d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e5c expref: 619 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.927276:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.927286:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000085e8ed28/0xd5401cbabc0208ed lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x69d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e5c expref: 619 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.927303:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x69b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.927309:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000085e8ed28/0xd5401cbabc0208ed lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x69d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e5c expref: 618 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.927312:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x69b:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.927323:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351635aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+618:2389:x1870153002416256:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 271us (668us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.927329:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025c14 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.927392:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36048 00010000:00010000:2.0:1783520935.927402:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025c14 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x69b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.927418:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x69d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.927425:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x69d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.927432:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025c1b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.927434:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.927440:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36051 00010000:00010000:3.0:1783520935.927450:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025c1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x69d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.927458:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516358700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+616:2387:x1870153002416640:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.927470:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f0050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+616:2389:x1870153002416384:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 399us (785us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.927472:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520935.927485:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36049 00010000:00010000:3.0:1783520935.927486:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.927487:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000805870f0/0xd5401cbabc0208df lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x69c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e4e expref: 616 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.927527:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000805870f0/0xd5401cbabc0208df lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x69c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e4e expref: 615 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.927545:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516359880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+615:2388:x1870153002416512:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 379us (671us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.927551:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000805870f0/0xd5401cbabc0208df lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x69c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e4e expref: 615 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.927572:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x69c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.927577:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x69c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.927582:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025c0d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.927591:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025c0d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x69c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.927594:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d2ad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993150464:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.927606:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025c14 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.927609:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36050 00010000:00010000:0.0:1783520935.927640:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025c29 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x69f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.927644:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025c14 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.927659:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025c06 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.927659:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.927711:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025c14 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.927712:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025c06 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.927767:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025c14 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x69b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.927767:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516358700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+613:2387:x1870153002416640:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 237us (784us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.927767:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.927769:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025c06 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.927824:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025c06 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x69a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.927829:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36051 00010000:00010000:2.0:1783520935.927832:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.927861:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520935.927866:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00000100:00100000:1.0:1783520935.927873:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002416768 00010000:00010000:2.0:1783520935.927876:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025c14 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x69b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.927879:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.927882:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002416768 00010000:00010000:2.0:1783520935.927896:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbaa00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.927903:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025c14 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x69b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.927913:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001bfd0fc9/0xd5401cbabc0229b6 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x69e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30a6 expref: 613 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.927917:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025c14 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x69b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.927931:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.927971:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025c14 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x69b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.927978:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002416896 00010000:00010000:2.0:1783520935.927982:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025c14 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x69b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.927985:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002416896 00010000:00010000:1.0:1783520935.928013:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000406c3132/0xd5401cbabc0208fb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x69f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e78 expref: 614 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.928020:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.928027:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025c06 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x69a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.928034:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002417024 00000100:00100000:0.0:1783520935.928035:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.928047:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5200 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520935.928047:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002417024 00010000:00010000:2.0:1783520935.928057:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025c06 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x69a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.928068:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025c06 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x69a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.928076:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d8bd7620/0xd5401cbabc020902 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e86 expref: 615 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.928087:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d28380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993150720:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.928088:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025c06 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x69a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.928102:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d28a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+391:3628:x1870152993149056:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6389us (7059us total) trans 4294975958 rc 0/0 00000100:00100000:1.0:1783520935.928127:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002417152 00010000:00010000:0.0:1783520935.928127:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025c30 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.928137:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002417152 00010000:00010000:2.0:1783520935.928143:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025c06 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x69a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.928156:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44456 00010000:00010000:1.0:1783520935.928174:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000fe5852fe/0xd5401cbabc02091e lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e94 expref: 616 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.928215:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36052 00000100:00100000:3.0:1783520935.928228:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36053 00000100:00100000:1.0:1783520935.928231:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436342b9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+616:2387:x1870153002416768:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.928233:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520935.928233:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943509a81c00/2, rc 0, transno 4294975958 00010000:00010000:2.0:1783520935.928239:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.928246:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.928247:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d2ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+616:2390:x1870153002416896:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.928257:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a81c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993149056:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.928259:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.928260:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.928261:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.928283:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001bfd0fc9/0xd5401cbabc0229b6 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x69e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30a6 expref: 616 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.928284:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000406c3132/0xd5401cbabc0208fb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x69f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e78 expref: 616 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.928299:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36054 00010000:00010000:3.0:1783520935.928331:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000406c3132/0xd5401cbabc0208fb lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x69f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e78 expref: 616 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.928340:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d2bb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+616:2387:x1870153002417024:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.928342:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001bfd0fc9/0xd5401cbabc0229b6 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x69e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30a6 expref: 616 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.928391:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993151104:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.928398:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.928414:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d8bd7620/0xd5401cbabc020902 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e86 expref: 616 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.928427:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x69e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.928434:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d8bd7620/0xd5401cbabc020902 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6a0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e86 expref: 616 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.928436:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x69e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.928454:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025c22 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.928456:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.928485:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025c22 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x69e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.928487:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d8bd7620/0xd5401cbabc020902 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6a0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e86 expref: 616 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.928515:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000406c3132/0xd5401cbabc0208fb lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x69f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e78 expref: 616 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.928549:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.928556:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.928559:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x69f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.928559:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.928564:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025c30 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.928567:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x69f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.928587:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025c29 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.928589:0:35275:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001bfd0fc9/0xd5401cbabc0229b6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x69e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30a6 expref: 614 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.928595:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025c30 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.928617:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436342b9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+614:2387:x1870153002416768:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 381us (855us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.928664:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025c29 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x69f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.928675:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36052 00000100:00100000:0.0:1783520935.928675:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351635b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993150208:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.928685:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.928713:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025c22 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.928714:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.928730:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d2bb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+613:2387:x1870153002417024:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 406us (806us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.928730:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025c22 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.928746:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025c22 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.928752:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36054 00000100:00100000:3.0:1783520935.928762:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d2ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+613:2390:x1870153002416896:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 515us (877us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.928768:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025c22 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.928805:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36055 00010000:00010000:0.0:1783520935.928806:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025c22 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x69e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.928822:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36053 00000100:00100000:2.0:1783520935.928828:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350ade3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+611:2387:x1870153002417152:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.928830:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.928844:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.928852:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000fe5852fe/0xd5401cbabc02091e lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6a1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e94 expref: 610 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.928859:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025c29 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.928868:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000fe5852fe/0xd5401cbabc02091e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e94 expref: 610 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.928881:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025c29 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.928927:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516359180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993150976:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.928938:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025c29 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.928950:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025c29 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x69f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.928952:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000fe5852fe/0xd5401cbabc02091e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6a1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0e94 expref: 610 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.928956:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025c37 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6a1:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.929009:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.929016:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a1:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.929019:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d29180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+391:3628:x1870152993149824:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6171us (6284us total) trans 4294975959 rc 0/0 00010000:00010000:3.0:1783520935.929025:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520935.929025:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025c37 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.929033:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44460 00010000:00010000:2.0:1783520935.929049:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025c37 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.929057:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993151104 00000100:00100000:0.0:1783520935.929062:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993151104 00010000:00010000:2.0:1783520935.929071:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520935.929078:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025c22 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x69e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.929089:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbbe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.929094:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025c22 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x69e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.929107:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025c22 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x69e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.929110:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44464 00000100:00100000:0.0:1783520935.929125:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350ade3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+392:3628:x1870152993151104:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.929131:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025c22 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x69e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.929146:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025c22 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x69e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.929149:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1698 count 1 00010000:00010000:0.0:1783520935.929175:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025c3e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6a2:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.929192:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.929192:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000cd201d10/0xd5401cbabc020909 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ea2 expref: 609 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.929203:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025c29 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x69f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.929206:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d29f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+392:3626:x1870152993150336:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5643us (5840us total) trans 4294975960 rc 0/0 00010000:00010000:0.0:1783520935.929220:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a2:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.929240:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a2:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.929254:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44461 00010000:00010000:2.0:1783520935.929256:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.929259:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025c3e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.929265:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025c29 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x69f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.929278:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025c29 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x69f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.929292:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000cd201d10/0xd5401cbabc020909 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ea2 expref: 609 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.929296:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025c29 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x69f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.929298:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025c1b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.929306:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000cd201d10/0xd5401cbabc020909 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ea2 expref: 609 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.929326:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025c29 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x69f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.929331:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993151232:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.929393:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025c1b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.929401:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.929406:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.929407:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025c1b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.929410:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.929430:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025c1b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x69d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.929440:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.929446:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470900 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.929452:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.929461:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.929472:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025c1b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x69d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.929475:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350ade3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+609:2387:x1870153002417152:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 654us (1446us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.929516:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb800 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520935.929517:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36055 00000004:00080000:0.0:1783520935.929522:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514d28e00/2, rc 0, transno 4294975960 00010000:00010000:1.0:1783520935.929531:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025c1b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x69d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.929540:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993150336:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.929547:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025c1b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x69d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.929575:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025c1b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x69d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.929583:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025c1b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x69d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.929589:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d29c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+392:3627:x1870152993149568:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6996us (7241us total) trans 4294975961 rc 0/0 00000004:00080000:2.0:1783520935.929590:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514d2b480/2, rc 0, transno 4294975961 00000100:00100000:2.0:1783520935.929609:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993149568:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.929617:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44459 00010000:00010000:1.0:1783520935.929638:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.929647:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.929652:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.929661:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.929661:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993151360:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.929672:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025c37 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.929681:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025c0d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.929684:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025c37 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.929703:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025c0d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.929706:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025c37 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.929721:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.929741:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025c0d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x69c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.929743:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025c37 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.929755:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:3.0:1783520935.929771:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025c0d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x69c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.929772:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993151488:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.929808:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d2a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+392:3626:x1870152993149184:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7596us (7708us total) trans 4294975962 rc 0/0 00000100:00100000:0.0:1783520935.929828:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.929836:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44458 00010000:00010000:1.0:1783520935.929847:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520935.929856:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520935.929872:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520935.929881:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025c37 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.929896:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514d2a300/2, rc 0, transno 4294975962 00010000:00010000:2.0:1783520935.929897:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.929906:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025c37 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.929913:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993149184:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.929920:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025c37 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.929943:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025c37 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.929952:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025c37 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.929998:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520935.929998:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993151616:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.929999:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436342ba300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+393:3628:x1870152993150848:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4993us (5113us total) trans 4294975963 rc 0/0 00010000:00010000:2.0:1783520935.930014:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025c0d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x69c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.930017:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d28000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+393:3626:x1870152993149440:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7926us (8038us total) trans 4294975964 rc 0/0 00000100:00100000:1.0:1783520935.930034:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44463 00000100:00100000:3.0:1783520935.930051:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44457 00010000:00010000:2.0:1783520935.930052:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.930063:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025c0d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x69c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.930069:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993151360 00000100:00100000:0.0:1783520935.930069:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.930074:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025c0d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x69c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.930082:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993151360 00000100:00100000:3.0:1783520935.930091:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993151488 00010000:00010000:2.0:1783520935.930091:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025c0d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x69c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.930104:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993151488 00010000:00010000:2.0:1783520935.930104:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025c0d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x69c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.930153:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.930158:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520935.930159:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514d2b100/2, rc 0, transno 4294975964 00000100:00100000:1.0:1783520935.930160:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993151616 00010000:00010000:2.0:1783520935.930161:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.930172:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44465 00010000:00010000:2.0:1783520935.930173:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.930173:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993151616 00000100:00100000:0.0:1783520935.930179:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993149440:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.930192:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635603050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+394:3625:x1870152993151360:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.930201:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44466 00000100:00100000:2.0:1783520935.930211:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350ade1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+394:3625:x1870152993151488:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520935.930224:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1699 count 1 00002000:00080000:2.0:1783520935.930231:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1700 count 1 00000100:00100000:1.0:1783520935.930233:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44467 00000100:00100000:1.0:1783520935.930253:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350ade0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+394:3627:x1870152993151616:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.930255:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:2.0:1783520935.930258:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025c45 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6a4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.930266:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025c4c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6a3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.930273:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993151744:0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.930284:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1701 count 1 00010000:00010000:2.0:1783520935.930303:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000537a004c/0xd5401cbabc020917 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ebe expref: 608 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.930311:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a4:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.930313:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025c53 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6a5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.930314:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c6e2bbc7/0xd5401cbabc020910 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0eb0 expref: 608 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.930316:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.930334:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025c45 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.930348:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.930403:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a3:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.930409:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.930409:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009a7529eb/0xd5401cbabc020925 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ecc expref: 608 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.930419:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025c4c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.930423:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a5:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.930430:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.930436:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025c53 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.930467:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c6e2bbc7/0xd5401cbabc020910 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0eb0 expref: 608 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.930473:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009a7529eb/0xd5401cbabc020925 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ecc expref: 608 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.930478:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514d2aa00/2, rc 0, transno 4294975963 00000100:00100000:0.0:1783520935.930513:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993150848:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.930548:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000537a004c/0xd5401cbabc020917 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ebe expref: 608 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.930551:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c6e2bbc7/0xd5401cbabc020910 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0eb0 expref: 608 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.930551:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009a7529eb/0xd5401cbabc020925 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ecc expref: 608 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.930554:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514d2b800/2, rc 0, transno 4294975959 00000100:00100000:0.0:1783520935.930577:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993149824:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.930621:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000537a004c/0xd5401cbabc020917 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ebe expref: 608 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.930628:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d320000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993151872:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.930628:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516359180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993152000:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.930670:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350805b100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993152128:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.930677:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993152256:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.930724:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d83c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.930724:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.930727:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435126766c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.930741:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.930777:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993151744 00000100:00100000:3.0:1783520935.930780:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993152256 00000100:00100000:1.0:1783520935.930785:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993151744 00010000:00010000:2.0:1783520935.930787:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025c30 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.930788:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993152256 00000100:00100000:0.0:1783520935.930791:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:2.0:1783520935.930803:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025c30 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.930804:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993152384:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.930823:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025c30 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.930832:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44468 00000100:00100000:0.0:1783520935.930852:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.930861:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44469 00000100:00100000:1.0:1783520935.930865:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350ade1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+396:3627:x1870152993151744:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.930874:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025c30 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.930889:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1702 count 1 00000100:00100000:3.0:1783520935.930892:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94360550c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+396:3625:x1870152993152256:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.930899:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993152384 00000100:00100000:0.0:1783520935.930912:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993152384 00010000:00010000:1.0:1783520935.930917:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025c5a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6a6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520935.930941:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1703 count 1 00010000:00010000:1.0:1783520935.930943:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a56fb236/0xd5401cbabc0228a5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f95 expref: 608 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.930944:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.930958:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44470 00010000:00010000:3.0:1783520935.930959:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025c61 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6a7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.930963:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.930971:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x6a6:0x0].0x0 since=36s v=(0 0) 00010000:00010000:2.0:1783520935.930973:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025c30 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.930986:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94360550ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+397:3625:x1870152993152384:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.930993:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.931023:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025c5a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.931024:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbebe00 already unused, calling callback (ffffffffc0bf9e90) 00002000:00080000:0.0:1783520935.931025:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1704 count 1 00010000:00010000:3.0:1783520935.931031:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009092bd79/0xd5401cbabc020933 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ee8 expref: 608 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.931038:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025c30 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.931052:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025c68 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6a8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.931101:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025c30 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.931105:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a7:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.931134:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.931135:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000cd1de2f9/0xd5401cbabc02093a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ef6 expref: 608 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.931146:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a56fb236/0xd5401cbabc0228a5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f95 expref: 608 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.931153:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025c30 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.931165:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025c61 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.931217:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a8:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.931221:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025c30 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.931226:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a56fb236/0xd5401cbabc0228a5 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f95 expref: 608 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.931238:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.931259:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009092bd79/0xd5401cbabc020933 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ee8 expref: 608 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.931261:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025c68 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.931294:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.931303:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.931311:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.931331:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009092bd79/0xd5401cbabc020933 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ee8 expref: 608 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.931334:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516358a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993152512:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.931353:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000cd1de2f9/0xd5401cbabc02093a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ef6 expref: 608 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.931355:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d321f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993152640:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.931396:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000cd1de2f9/0xd5401cbabc02093a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ef6 expref: 608 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.931423:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94360550e300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993152768:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.931434:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e840 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.931453:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8c00 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.931472:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470e40 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.931513:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d29880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+398:3626:x1870152993150592:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7307us (7414us total) trans 4294975965 rc 0/0 00000100:00100000:2.0:1783520935.931525:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44462 00000004:00080000:0.0:1783520935.931535:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514d29500/2, rc 0, transno 4294975965 00000100:00100000:0.0:1783520935.931550:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993150592:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.931632:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94360550f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993152896:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.931667:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.931712:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993152896 00000100:00100000:0.0:1783520935.931717:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993152896 00000100:00100000:0.0:1783520935.931746:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44471 00000100:00100000:0.0:1783520935.931752:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94360550ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+398:3627:x1870152993152896:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.931765:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1705 count 1 00010000:00010000:0.0:1783520935.931780:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025c6f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6a9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.931792:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000020799d5d/0xd5401cbabc020941 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f04 expref: 608 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.931801:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a9:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.931804:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.931809:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025c6f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.931832:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000020799d5d/0xd5401cbabc020941 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f04 expref: 608 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.931843:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000020799d5d/0xd5401cbabc020941 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f04 expref: 608 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.931854:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94360550e680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993153024:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.931889:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470b40 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.932628:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.932668:0:35277:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002417280 00000100:00100000:1.0:1783520935.932675:0:35277:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002417280 00000100:00100000:1.0:1783520935.932727:0:35277:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98414 00000100:00100000:3.0:1783520935.932735:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993151232:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.932740:0:35277:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508059880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_008:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+36:251806:x1870153002417280:12345-192.168.204.33@tcp:101:stat.0 00010000:00010000:1.0:1783520935.932760:0:35277:0:(ldlm_lockd.c:1265:ldlm_handle_enqueue()) ### server-side enqueue handler START 00010000:00010000:1.0:1783520935.932777:0:35277:0:(ldlm_lockd.c:1343:ldlm_handle_enqueue()) ### server-side enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943513bba600/0xd5401cbabc025c76 lrc: 2/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT gid 0 flags: 0x40000000000000 nid: local remote: 0xc275876b859f3506 expref: -99 pid: 35277 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.932800:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025c3e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.932877:0:35277:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff943513bba200/0xd5401cbabc025c7d lrc: 3/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x40000000000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.932891:0:35277:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943513bba200/0xd5401cbabc025c7d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT flags: 0x50210000000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.932903:0:35277:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943513bba200/0xd5401cbabc025c7d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00000004:00200000:1.0:1783520935.932948:0:35277:0:(mdt_handler.c:909:mdt_pack_attr2body()) [0x200000007:0x1:0x0]: returning size 69632 00000004:00200000:1.0:1783520935.932951:0:35277:0:(mdt_handler.c:1574:mdt_getattr_internal()) dirent count 0 stripe count 1 MDT count 2 00000004:00010000:1.0:1783520935.932977:0:35277:0:(mdt_handler.c:2507:mdt_getattr_name_lock()) ### Returning lock to client ns: mdt-lustre-MDT0000_UUID lock: ffff943513bba200/0xd5401cbabc025c7d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT flags: 0x40210000000000 pid: 35277 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.933005:0:35277:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943513bba600/0xd5401cbabc025c76 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x44000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3506 expref: 38 pid: 35277 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.933030:0:35277:0:(ldlm_lockd.c:1513:ldlm_handle_enqueue()) ### server-side enqueue handler, sending reply (err=0, rc=0) ns: mdt-lustre-MDT0000_UUID lock: ffff943513bba200/0xd5401cbabc025c7d lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x40200000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3506 expref: 37 pid: 35277 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.933040:0:35277:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x0 (ffff943504b72c00) 00010000:00010000:1.0:1783520935.933043:0:35277:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 13 00010000:00010000:1.0:1783520935.933047:0:35277:0:(ldlm_lockd.c:1592:ldlm_handle_enqueue()) ### server-side enqueue handler END (lock ffff943513bba200, rc 0) 00000100:00100000:0.0:1783520935.933056:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350805b100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993152128:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.933060:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516359180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993152000:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.933083:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025c45 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.933085:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025c53 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.933102:0:35277:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508059880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_008:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+37:251806:x1870153002417280:12345-192.168.204.33@tcp:101:stat.0 Request processed in 366us (477us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.933119:0:35277:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98414 00000100:00100000:1.0:1783520935.933459:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516358a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993152512:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.933474:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025c5a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.933523:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d320000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993151872:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.933540:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025c4c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.934608:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.934660:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94360550e680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993153024:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.934682:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025c6f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.934695:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002417536 00000100:00100000:1.0:1783520935.934701:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002417536 00010000:00010000:1.0:1783520935.934728:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000537a004c/0xd5401cbabc020917 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ebe expref: 609 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.934758:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36056 00000100:00100000:1.0:1783520935.934765:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f6c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+609:2387:x1870153002417536:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.934776:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.934781:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000537a004c/0xd5401cbabc020917 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ebe expref: 609 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.934792:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000537a004c/0xd5401cbabc020917 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6a4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ebe expref: 609 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.934816:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000537a004c/0xd5401cbabc020917 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6a4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ebe expref: 609 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.934828:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.934832:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.934842:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025c45 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.934852:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025c45 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.934878:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.934902:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025c45 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.934913:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025c45 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.934924:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025c45 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.934932:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025c45 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.934956:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520935.934985:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d321f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993152640:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.934985:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.935008:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.935012:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025c61 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.935015:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025c45 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.935026:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.935033:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025c45 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.935043:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025c45 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.935051:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002417664 00010000:00010000:2.0:1783520935.935055:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025c45 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.935056:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002417664 00010000:00010000:2.0:1783520935.935066:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025c45 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.935082:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009a7529eb/0xd5401cbabc020925 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ecc expref: 609 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.935101:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.935104:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.935107:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.935113:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36057 00000100:00100000:0.0:1783520935.935122:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350ade1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+399:3625:x1870152993151488:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4912us (5307us total) trans 4294975966 rc 0/0 00000100:00100000:3.0:1783520935.935130:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350805b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+609:2387:x1870153002417664:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.935137:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44466 00010000:00010000:3.0:1783520935.935142:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520935.935142:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.935152:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009a7529eb/0xd5401cbabc020925 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ecc expref: 609 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.935168:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002417408 00010000:00010000:3.0:1783520935.935173:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009a7529eb/0xd5401cbabc020925 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6a5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ecc expref: 609 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.935174:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002417408 00000004:00080000:0.0:1783520935.935184:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350ade3b80/2, rc 0, transno 4294975966 00010000:00010000:2.0:1783520935.935194:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000cd201d10/0xd5401cbabc020909 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ea2 expref: 610 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.935203:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009a7529eb/0xd5401cbabc020925 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6a5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ecc expref: 610 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.935203:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993151488:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.935226:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.935229:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.935234:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025c53 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.935237:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36058 00000100:00100000:2.0:1783520935.935248:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508059f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+609:2387:x1870153002417408:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.935250:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025c53 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.935259:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.935265:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000cd201d10/0xd5401cbabc020909 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ea2 expref: 609 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.935274:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.935279:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000cd201d10/0xd5401cbabc020909 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6a2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ea2 expref: 609 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.935295:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cd201d10/0xd5401cbabc020909 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6a2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ea2 expref: 609 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.935302:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94360550e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993153152:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.935308:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a2:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.935312:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350805b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+608:2387:x1870153002417664:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 189us (327us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.935313:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.935322:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025c3e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.935327:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36057 00010000:00010000:2.0:1783520935.935336:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025c3e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.935342:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.935387:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.935413:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508059f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+607:2387:x1870153002417408:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 165us (276us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.935421:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36058 00000100:00100000:0.0:1783520935.935436:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993153152 00000100:00100000:0.0:1783520935.935441:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993153152 00000100:00100000:2.0:1783520935.935472:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.935475:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025c53 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.935477:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44472 00010000:00010000:3.0:1783520935.935488:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025c53 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.935489:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94360550c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+399:3628:x1870152993153152:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.935510:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025c53 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.935520:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025c53 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.935540:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.935545:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00002000:00080000:0.0:1783520935.935545:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1706 count 1 00010000:00010000:2.0:1783520935.935553:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025c53 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.935562:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.935566:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025c53 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.935570:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025c84 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6aa:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.935580:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025c53 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.935595:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025c53 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.935603:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025c53 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.935619:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.935621:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.935623:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.935634:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002417792 00000100:00100000:2.0:1783520935.935639:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002417792 00010000:00010000:2.0:1783520935.935654:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a56fb236/0xd5401cbabc0228a5 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f95 expref: 607 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.935679:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36059 00010000:00010000:0.0:1783520935.935684:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003c6d306d/0xd5401cbabc020948 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f12 expref: 607 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.935692:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350805ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+607:2387:x1870153002417792:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520935.935695:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350ade0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+400:3627:x1870152993151616:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5448us (5637us total) trans 4294975967 rc 0/0 00010000:00010000:0.0:1783520935.935703:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6aa:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.935708:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520935.935710:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44467 00010000:00010000:0.0:1783520935.935711:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6aa:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.935716:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a56fb236/0xd5401cbabc0228a5 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6a6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f95 expref: 607 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.935720:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025c84 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.935737:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a56fb236/0xd5401cbabc0228a5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6a6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f95 expref: 607 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.935739:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025c3e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.935756:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003c6d306d/0xd5401cbabc020948 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f12 expref: 607 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.935761:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025c3e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.935764:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a56fb236/0xd5401cbabc0228a5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6a6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f95 expref: 607 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.935780:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003c6d306d/0xd5401cbabc020948 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f12 expref: 607 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.935789:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025c3e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.935792:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.935799:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a6:0x0].0x0 since=0s v=(1 0) 00000100:00100000:0.0:1783520935.935804:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94360550d880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993153280:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.935809:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025c5a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.935817:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025c3e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.935832:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025c5a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.935847:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.935853:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520935.935858:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.935860:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025c3e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.935869:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.935873:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025c3e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.935881:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350ade1500/2, rc 0, transno 4294975967 00010000:00010000:2.0:1783520935.935892:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025c3e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.935896:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993151616:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.935912:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025c3e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.935921:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025c3e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.935929:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350ade3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+400:3628:x1870152993151104:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6808us (7479us total) trans 4294975968 rc 0/0 00000100:00100000:3.0:1783520935.935936:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44464 00010000:00010000:2.0:1783520935.935947:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.935951:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.935955:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520935.935967:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350ade0a80/2, rc 0, transno 4294975968 00010000:00010000:2.0:1783520935.935974:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.935980:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993151104:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.936002:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350805ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+606:2387:x1870153002417792:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 313us (531us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.936012:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36059 00000100:00100000:0.0:1783520935.936020:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94360550dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993153536:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.936045:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.936055:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.936082:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.936084:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1783520935.936093:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94360550df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993153408:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.936102:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002417920 00000100:00100000:2.0:1783520935.936107:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002417920 00010000:00010000:2.0:1783520935.936123:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c6e2bbc7/0xd5401cbabc020910 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0eb0 expref: 606 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.936124:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.936134:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002418048 00000100:00100000:3.0:1783520935.936140:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002418048 00000100:00100000:2.0:1783520935.936152:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36060 00010000:00010000:3.0:1783520935.936160:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009092bd79/0xd5401cbabc020933 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ee8 expref: 607 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.936165:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c3800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+607:2390:x1870153002417920:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.936169:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993153536 00000100:00100000:0.0:1783520935.936175:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993153536 00010000:00010000:2.0:1783520935.936183:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.936189:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c6e2bbc7/0xd5401cbabc020910 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0eb0 expref: 607 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.936200:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c6e2bbc7/0xd5401cbabc020910 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6a3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0eb0 expref: 607 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.936205:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993153408 00000100:00100000:0.0:1783520935.936211:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993153408 00010000:00010000:2.0:1783520935.936215:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c6e2bbc7/0xd5401cbabc020910 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6a3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0eb0 expref: 607 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.936234:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44473 00010000:00010000:2.0:1783520935.936240:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.936244:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a3:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.936247:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94360550ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+401:3626:x1870152993153536:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.936250:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025c4c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.936258:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36061 00002000:00080000:0.0:1783520935.936266:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1708 count 1 00010000:00010000:2.0:1783520935.936267:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025c4c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.936271:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c3100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+606:2387:x1870153002418048:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.936284:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025c8b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6ac:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.936285:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.936292:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009092bd79/0xd5401cbabc020933 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ee8 expref: 606 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.936296:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.936306:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008b766929/0xd5401cbabc020956 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f2e expref: 606 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.936308:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009092bd79/0xd5401cbabc020933 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6a7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ee8 expref: 606 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.936325:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c3800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+606:2390:x1870153002417920:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 163us (278us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.936329:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ac:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.936335:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ac:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.936336:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36060 00010000:00010000:3.0:1783520935.936337:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009092bd79/0xd5401cbabc020933 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6a7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ee8 expref: 606 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.936342:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025c8b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.936384:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.936389:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.936391:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008b766929/0xd5401cbabc020956 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f2e expref: 604 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.936395:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025c61 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.936398:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44474 00000100:00100000:2.0:1783520935.936411:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94360550d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+401:3626:x1870152993153408:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.936421:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008b766929/0xd5401cbabc020956 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f2e expref: 604 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.936424:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025c61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.936431:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1707 count 1 00000100:00100000:0.0:1783520935.936446:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94360550f100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993153664:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.936448:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025c92 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6ab:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.936456:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.936465:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000047110ec6/0xd5401cbabc02094f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f20 expref: 604 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.936472:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ab:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.936476:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ab:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.936479:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025c92 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.936487:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470b40 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.936506:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c3100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+604:2387:x1870153002418048:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 224us (409us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.936518:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36061 00010000:00010000:2.0:1783520935.936519:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000047110ec6/0xd5401cbabc02094f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f20 expref: 604 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.936534:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000047110ec6/0xd5401cbabc02094f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f20 expref: 603 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.936546:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c3b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993153792:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.936548:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025c61 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.936558:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025c61 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.936566:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025c61 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.936574:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025c61 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.936576:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676840 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.936587:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f6c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+603:2387:x1870153002417536:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 1819us (1978us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.936596:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.936599:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36056 00010000:00010000:3.0:1783520935.936610:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.936616:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025c61 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.936624:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.936631:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.936635:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025c61 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.936642:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025c61 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.936656:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025c61 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.936658:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002418176 00000100:00100000:1.0:1783520935.936664:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002418176 00010000:00010000:3.0:1783520935.936675:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025c61 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.936685:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000020799d5d/0xd5401cbabc020941 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f04 expref: 603 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.936702:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.936706:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.936711:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.936711:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94360550e300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993152768:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.936726:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36062 00000100:00100000:2.0:1783520935.936727:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.936732:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025c68 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.936743:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c1c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+603:2390:x1870153002418176:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.936753:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520935.936754:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002418304 00000100:00100000:2.0:1783520935.936760:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002418304 00010000:00010000:1.0:1783520935.936761:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000020799d5d/0xd5401cbabc020941 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f04 expref: 603 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.936775:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000020799d5d/0xd5401cbabc020941 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6a9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f04 expref: 604 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.936780:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000cd1de2f9/0xd5401cbabc02093a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ef6 expref: 604 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.936798:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000020799d5d/0xd5401cbabc020941 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6a9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f04 expref: 604 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.936809:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94360550c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+402:3625:x1870152993152256:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5930us (6071us total) trans 4294975969 rc 0/0 00010000:00010000:1.0:1783520935.936812:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a9:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.936814:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36063 00010000:00010000:1.0:1783520935.936820:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a9:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.936826:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44469 00010000:00010000:1.0:1783520935.936826:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025c6f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.936830:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350ade2300/2, rc 0, transno 4294975969 00000100:00100000:2.0:1783520935.936831:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f2050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+603:2388:x1870153002418304:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.936843:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993152256:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.936847:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025c6f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.936848:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.936857:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025c4c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.936859:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000cd1de2f9/0xd5401cbabc02093a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6a8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ef6 expref: 603 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.936874:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.936880:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025c4c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.936886:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000cd1de2f9/0xd5401cbabc02093a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6a8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ef6 expref: 603 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.936895:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025c4c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.936898:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1783520935.936904:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c1c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+603:2390:x1870153002418176:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 163us (282us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.936908:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cd1de2f9/0xd5401cbabc02093a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6a8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ef6 expref: 603 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.936912:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025c4c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.936912:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94360550e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993153920:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.936925:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36062 00010000:00010000:2.0:1783520935.936940:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.936945:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6a8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.936946:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.936953:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.936955:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025c68 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.936958:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025c5a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.936974:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025c68 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.936977:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025c5a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.936991:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993153920 00000100:00100000:0.0:1783520935.936997:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993153920 00010000:00010000:1.0:1783520935.937000:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025c5a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.937006:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.937013:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025c5a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.937034:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f2050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+601:2388:x1870153002418304:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 206us (310us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.937037:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635603050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+403:3625:x1870152993151360:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6853us (7324us total) trans 4294975970 rc 0/0 00010000:00010000:1.0:1783520935.937037:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:0.0:1783520935.937038:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44475 00000100:00100000:2.0:1783520935.937047:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36063 00000100:00100000:3.0:1783520935.937055:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44465 00000100:00100000:0.0:1783520935.937055:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94360550c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+403:3628:x1870152993153920:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.937073:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1709 count 1 00010000:00010000:2.0:1783520935.937079:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520935.937085:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025c4c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.937087:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025c99 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6ad:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.937114:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.937116:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007ad9f65d/0xd5401cbabc02095d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f3c expref: 600 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.937120:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025c4c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.937134:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ad:0x0].0x0 since=37s v=(0 0) 00000100:00100000:1.0:1783520935.937138:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350ade1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+403:3627:x1870152993151744:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6286us (6807us total) trans 4294975971 rc 0/0 00010000:00010000:2.0:1783520935.937140:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025c4c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.937140:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ad:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.937151:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44468 00010000:00010000:0.0:1783520935.937152:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025c99 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.937169:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025c4c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.937179:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025c4c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.937187:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007ad9f65d/0xd5401cbabc02095d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f3c expref: 600 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.937189:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350ade2d80/2, rc 0, transno 4294975971 00010000:00010000:2.0:1783520935.937198:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.937204:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007ad9f65d/0xd5401cbabc02095d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f3c expref: 600 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.937205:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993151744:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.937206:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025c5a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.937230:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbba00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520935.937235:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350384bb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993154048:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.937236:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025c5a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.937251:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516359880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993154176:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.937252:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025c5a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.937267:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025c5a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.937273:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470e40 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.937276:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025c5a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.937281:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.937300:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.937303:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520935.937306:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350ade0e00/2, rc 0, transno 4294975970 00010000:00010000:2.0:1783520935.937307:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.937317:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.937322:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993151360:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.937344:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025c68 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.937354:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025c68 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.937394:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025c68 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.937402:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025c68 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.937417:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.937418:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993154176 00000100:00100000:1.0:1783520935.937422:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993154176 00010000:00010000:0.0:1783520935.937425:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.937440:0:35278:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002418432 00000100:00100000:1.0:1783520935.937444:0:35278:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002418432 00010000:00010000:2.0:1783520935.937448:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.937454:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025c68 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.937462:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.937466:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025c68 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.937473:0:35278:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98415 00010000:00010000:2.0:1783520935.937479:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025c68 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.937488:0:35278:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_009:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+37:251806:x1870153002418432:12345-192.168.204.33@tcp:101:stat.0 00010000:00010000:2.0:1783520935.937508:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025c68 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.937512:0:35278:0:(ldlm_lockd.c:1265:ldlm_handle_enqueue()) ### server-side enqueue handler START 00010000:00010000:2.0:1783520935.937521:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025c68 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.937529:0:35278:0:(ldlm_lockd.c:1343:ldlm_handle_enqueue()) ### server-side enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943513bba600/0xd5401cbabc025ca0 lrc: 2/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT gid 0 flags: 0x40000000000000 nid: local remote: 0xc275876b859f350d expref: -99 pid: 35278 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.937534:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94360550ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+404:3625:x1870152993152384:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6556us (6688us total) trans 4294975972 rc 0/0 00010000:00010000:2.0:1783520935.937539:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.937544:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.937548:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.937550:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44470 00000100:00100000:0.0:1783520935.937578:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1783520935.937586:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516358a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993154304:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.937611:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.937615:0:35278:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff943513bba000/0xd5401cbabc025ca7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 6 type: IBT flags: 0x40000000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.937629:0:35278:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943513bba000/0xd5401cbabc025ca7 lrc: 3/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x50210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.937639:0:35278:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943513bba000/0xd5401cbabc025ca7 lrc: 3/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x40210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.937648:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514d28380/2, rc 0, transno 4294975972 00010000:00010000:1.0:1783520935.937653:0:35278:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbae00/0xd5401cbabc025cae lrc: 3/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x40000000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.937662:0:35278:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbae00/0xd5401cbabc025cae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.937669:0:35278:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbae00/0xd5401cbabc025cae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.937672:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993152384:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.937697:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993154304 00000100:00100000:0.0:1783520935.937701:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993154304 00000100:00100000:0.0:1783520935.937732:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44476 00000100:00100000:0.0:1783520935.937740:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350384aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+404:3626:x1870152993154304:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.937754:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1711 count 1 00010000:00010000:1.0:1783520935.937762:0:35278:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbbc00/0xd5401cbabc025cb5 lrc: 3/1,0 mode: --/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x0/0x0 rrc: 8 type: IBT flags: 0x40000000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.937769:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025cbc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6af:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.937773:0:35278:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbbc00/0xd5401cbabc025cb5 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x5b/0x0 rrc: 8 type: IBT flags: 0x50210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.937789:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008395e9ef/0xd5401cbabc020964 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f58 expref: 600 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.937790:0:35278:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbbc00/0xd5401cbabc025cb5 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x5b/0x0 rrc: 8 type: IBT flags: 0x40210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.937804:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:0.0:1783520935.937815:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6af:0x0].0x0 since=37s v=(0 0) 00000100:00100000:2.0:1783520935.937816:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993154432:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.937823:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6af:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.937829:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025cbc lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00200000:1.0:1783520935.937845:0:35278:0:(mdt_handler.c:909:mdt_pack_attr2body()) [0x200000402:0x72a:0x0]: returning size 262144 00000100:00100000:2.0:1783520935.937852:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00200000:1.0:1783520935.937853:0:35278:0:(mdt_handler.c:1574:mdt_getattr_internal()) dirent count 0 stripe count 1 MDT count 2 00010000:00010000:0.0:1783520935.937857:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008395e9ef/0xd5401cbabc020964 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f58 expref: 600 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.937869:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008395e9ef/0xd5401cbabc020964 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f58 expref: 600 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00010000:1.0:1783520935.937873:0:35278:0:(mdt_handler.c:2507:mdt_getattr_name_lock()) ### Returning lock to client ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbbc00/0xd5401cbabc025cb5 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x5b/0x0 rrc: 8 type: IBT flags: 0x40210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.937883:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993154432 00000100:00100000:2.0:1783520935.937888:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993154432 00000100:00100000:0.0:1783520935.937889:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503848700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993154560:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.937890:0:35278:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbbc00/0xd5401cbabc025cb5 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 8 type: IBT flags: 0x50210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.937905:0:35278:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff943513bba000/0xd5401cbabc025ca7 lrc: 4/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x40210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.937917:0:35278:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff943513bba000/0xd5401cbabc025ca7 lrc: 4/1,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x50210400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.937922:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44477 00010000:00010000:1.0:1783520935.937928:0:35278:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff943513bba000/0xd5401cbabc025ca7 lrc: 3/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x50210400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.937937:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+405:3625:x1870152993154432:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.937939:0:35278:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff943513bba000/0xd5401cbabc025ca7 lrc: 4/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x40210400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.937948:0:35278:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba000 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:1.0:1783520935.937952:0:35278:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943513bba000/0xd5401cbabc025ca7 lrc: 4/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x40210400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.937954:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1712 count 1 00000100:00100000:0.0:1783520935.937958:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470900 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.937966:0:35278:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943513bba000/0xd5401cbabc025ca7 lrc: 5/0,0 mode: CR/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x40218400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.937970:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025cc3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6b0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.937978:0:35278:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x0 (ffff943504b72c00) 00010000:00010000:1.0:1783520935.937983:0:35278:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:2.0:1783520935.937985:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000098746591/0xd5401cbabc020972 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f66 expref: 600 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.937988:0:35278:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff943513bba000/0xd5401cbabc025ca7 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x44a19400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.938003:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b0:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.938006:0:35278:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943513bba000/0xd5401cbabc025ca7 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x44a19400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.938009:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.938019:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025cc3 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.938027:0:35278:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbae00/0xd5401cbabc025cae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210000000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.938035:0:35278:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbae00/0xd5401cbabc025cae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x50210400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.938041:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000098746591/0xd5401cbabc020972 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f66 expref: 600 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.938045:0:35278:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbae00/0xd5401cbabc025cae lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.938062:0:35278:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbae00 already unused, calling callback (ffffffffc17ee4f0) 00010000:00010000:2.0:1783520935.938066:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000098746591/0xd5401cbabc020972 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f66 expref: 600 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.938067:0:35278:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbae00/0xd5401cbabc025cae lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40210400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.938086:0:35278:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbae00/0xd5401cbabc025cae lrc: 4/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x40218400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.938106:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c0000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993154688:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.938107:0:35278:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000007:0x1:0x0].0x5ba6781 (ffff943507a8d500) 00010000:00010000:1.0:1783520935.938114:0:35278:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 2 00010000:00010000:1.0:1783520935.938117:0:35278:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbae00/0xd5401cbabc025cae lrc: 2/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.938123:0:35278:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbae00/0xd5401cbabc025cae lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x5ba6781 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x44a19400000000 pid: 35278 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.938146:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676540 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.938156:0:35278:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943513bba600/0xd5401cbabc025ca0 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x44000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f350d expref: 39 pid: 35278 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.938172:0:35278:0:(ldlm_lockd.c:1513:ldlm_handle_enqueue()) ### server-side enqueue handler, sending reply (err=0, rc=0) ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbbc00/0xd5401cbabc025cb5 lrc: 3/0,0 mode: PR/PR res: [0x200000402:0x72a:0x0].0x0 bits 0x1b/0x0 rrc: 8 type: IBT gid 0 flags: 0x40200000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f350d expref: 38 pid: 35278 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.938180:0:35278:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x72a:0x0].0x0 (ffff943515473e00) 00010000:00010000:1.0:1783520935.938183:0:35278:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint 1b 00010000:00010000:1.0:1783520935.938186:0:35278:0:(ldlm_lockd.c:1592:ldlm_handle_enqueue()) ### server-side enqueue handler END (lock ffff943513bbbc00, rc 0) 00000100:00100000:1.0:1783520935.938225:0:35278:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_009:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+38:251806:x1870153002418432:12345-192.168.204.33@tcp:101:stat.0 Request processed in 742us (810us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.938232:0:35278:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98415 00000100:00100000:1.0:1783520935.938284:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44478 00000100:00100000:1.0:1783520935.938291:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516358700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+405:3628:x1870152993154176:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.938304:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1710 count 1 00010000:00010000:1.0:1783520935.938317:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025cca lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6ae:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.938330:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000062a21c8d/0xd5401cbabc022aff lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f31ef expref: 600 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.938338:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x6ae:0x0].0x0 since=36s v=(0 0) 00010000:00010000:1.0:1783520935.938342:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ae:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.938346:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025cca lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.938400:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000062a21c8d/0xd5401cbabc022aff lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31ef expref: 600 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.938411:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000062a21c8d/0xd5401cbabc022aff lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31ef expref: 600 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.938423:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516359180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993154816:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.938434:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025c6f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.938446:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025c6f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.938456:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025c6f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.938464:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025c6f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.938467:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3eb40 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.938503:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.938515:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.938522:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025c6f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.938531:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.938535:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025c6f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.938543:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025c6f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.938555:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025c6f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.938562:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025c6f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.938578:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.938580:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.938582:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.938609:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94360550ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+406:3627:x1870152993152896:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6855us (6939us total) trans 4294975973 rc 0/0 00000100:00100000:0.0:1783520935.938617:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44471 00000004:00080000:1.0:1783520935.938661:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94360550f800/2, rc 0, transno 4294975973 00000100:00100000:1.0:1783520935.938673:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94360550f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993152896:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.938732:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993154944:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.938760:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.938793:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993154944 00000100:00100000:2.0:1783520935.938799:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993154944 00000100:00100000:2.0:1783520935.938828:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44479 00000100:00100000:2.0:1783520935.938836:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351635b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+406:3627:x1870152993154944:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.938847:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1713 count 1 00010000:00010000:2.0:1783520935.938859:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025cd1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6b1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.938873:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000034269690/0xd5401cbabc020979 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f74 expref: 600 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.938880:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b1:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.938884:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.938888:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025cd1 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.938910:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000034269690/0xd5401cbabc020979 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f74 expref: 600 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.938921:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000034269690/0xd5401cbabc020979 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f74 expref: 600 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.938931:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c2d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993155072:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.938967:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435126766c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.939006:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94360550d880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993153280:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.939008:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.939035:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025c84 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.939040:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002418560 00000100:00100000:0.0:1783520935.939046:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002418560 00010000:00010000:0.0:1783520935.939066:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003c6d306d/0xd5401cbabc020948 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f12 expref: 601 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.939095:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36064 00000100:00100000:0.0:1783520935.939106:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d321f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+601:2388:x1870153002418560:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.939116:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.939121:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003c6d306d/0xd5401cbabc020948 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6aa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f12 expref: 601 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.939132:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003c6d306d/0xd5401cbabc020948 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6aa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f12 expref: 601 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.939147:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003c6d306d/0xd5401cbabc020948 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6aa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f12 expref: 601 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.939158:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6aa:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.939162:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6aa:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.939167:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025c84 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.939177:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025c84 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.939204:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.939225:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025c84 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.939233:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025c84 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.939235:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d321f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+600:2388:x1870153002418560:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 131us (230us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.939247:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025c84 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.939253:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36064 00010000:00010000:1.0:1783520935.939257:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025c84 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.939275:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.939283:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.939290:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025c84 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.939299:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.939304:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025c84 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.939313:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025c84 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.939326:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025c84 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.939334:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025c84 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.939348:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.939350:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.939353:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520935.939418:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94360550e680/2, rc 0, transno 4294975974 00000100:00100000:1.0:1783520935.939422:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94360550c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+407:3628:x1870152993153152:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3933us (4081us total) trans 4294975974 rc 0/0 00000100:00100000:1.0:1783520935.939432:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44472 00000100:00100000:0.0:1783520935.939435:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94360550e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993153152:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.939447:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c3b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993153792:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.939465:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025c92 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.939522:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350384b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993155200:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.939551:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.939577:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993155200 00000100:00100000:0.0:1783520935.939582:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993155200 00000100:00100000:0.0:1783520935.939613:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44480 00000100:00100000:0.0:1783520935.939621:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516359f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+407:3628:x1870152993155200:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.939636:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1714 count 1 00010000:00010000:0.0:1783520935.939648:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025cd8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6b2:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.939658:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000058c6dac1/0xd5401cbabc020980 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f82 expref: 599 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.939663:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b2:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.939666:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b2:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.939671:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350384bb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993154048:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.939671:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025cd8 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.939693:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.939700:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000058c6dac1/0xd5401cbabc020980 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f82 expref: 599 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.939701:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025c99 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.939704:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94360550f100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993153664:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.939724:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000058c6dac1/0xd5401cbabc020980 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f82 expref: 599 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.939731:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.939734:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025c8b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.939752:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503849500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993155328:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.939762:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002418816 00000100:00100000:3.0:1783520935.939767:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.939768:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002418816 00010000:00010000:1.0:1783520935.939790:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000047110ec6/0xd5401cbabc02094f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f20 expref: 600 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.939794:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002418688 00000100:00100000:0.0:1783520935.939798:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435190663c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.939800:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002418688 00010000:00010000:2.0:1783520935.939822:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007ad9f65d/0xd5401cbabc02095d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f3c expref: 601 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.939822:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002418944 00000100:00100000:1.0:1783520935.939827:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002418944 00010000:00010000:1.0:1783520935.939846:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008b766929/0xd5401cbabc020956 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f2e expref: 602 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.939865:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36065 00000100:00100000:1.0:1783520935.939875:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36066 00000100:00100000:2.0:1783520935.939877:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d320000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+602:2388:x1870153002418816:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.939888:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.939889:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d322300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+602:2387:x1870153002418688:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.939895:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000047110ec6/0xd5401cbabc02094f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ab:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f20 expref: 602 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.939902:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.939911:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000047110ec6/0xd5401cbabc02094f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6ab:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f20 expref: 602 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.939912:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007ad9f65d/0xd5401cbabc02095d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ad:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f3c expref: 602 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.939942:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007ad9f65d/0xd5401cbabc02095d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6ad:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f3c expref: 602 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.939947:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000047110ec6/0xd5401cbabc02094f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6ab:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f20 expref: 602 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.939964:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ab:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.939965:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007ad9f65d/0xd5401cbabc02095d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6ad:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f3c expref: 601 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.939970:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ab:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.939972:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36067 00010000:00010000:2.0:1783520935.939982:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025c92 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.939989:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ad:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.939991:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d321180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+600:2390:x1870153002418944:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.939999:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ad:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.940002:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025c92 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.940006:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.940007:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025c99 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.940018:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008b766929/0xd5401cbabc020956 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ac:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f2e expref: 600 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.940024:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025c99 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.940030:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.940038:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008b766929/0xd5401cbabc020956 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6ac:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f2e expref: 600 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.940057:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.940058:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008b766929/0xd5401cbabc020956 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6ac:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f2e expref: 600 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.940065:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d320000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+600:2388:x1870153002418816:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 190us (372us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.940073:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025c99 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.940078:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ac:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.940081:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36065 00000100:00100000:1.0:1783520935.940088:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d322300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+599:2387:x1870153002418688:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 199us (360us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.940090:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ac:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.940096:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025c99 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.940101:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025c8b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.940107:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36066 00010000:00010000:2.0:1783520935.940112:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025c92 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.940118:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025c99 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.940125:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025c8b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.940140:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025c92 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.940147:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025c99 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.940160:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.940161:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025c92 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.940173:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025c92 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.940175:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520935.940185:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d321180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+597:2390:x1870153002418944:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 199us (419us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.940194:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36067 00010000:00010000:2.0:1783520935.940195:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.940201:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025c99 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.940209:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.940213:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025c99 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.940222:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025c99 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.940235:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025c99 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.940243:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025c99 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.940260:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.940263:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.940266:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.940273:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94360550c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+408:3628:x1870152993153920:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3222us (3320us total) trans 4294975975 rc 0/0 00010000:00010000:2.0:1783520935.940280:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025c8b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.940282:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44475 00000004:00080000:1.0:1783520935.940290:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94360550e300/2, rc 0, transno 4294975975 00010000:00010000:2.0:1783520935.940293:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025c8b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.940303:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025c8b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.940305:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94360550e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993153920:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.940319:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025c8b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.940337:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520935.940453:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94360550ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+408:3626:x1870152993153536:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4209us (4398us total) trans 4294975976 rc 0/0 00000100:00100000:2.0:1783520935.940460:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44473 00000100:00100000:2.0:1783520935.940486:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351635aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993155456:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.940524:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.940548:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993155456 00000100:00100000:0.0:1783520935.940553:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993155456 00010000:00010000:2.0:1783520935.940572:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:0.0:1783520935.940579:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44481 00000100:00100000:0.0:1783520935.940586:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+408:3625:x1870152993155456:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.940593:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00002000:00080000:0.0:1783520935.940599:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1715 count 1 00010000:00010000:1.0:1783520935.940600:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025c8b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.940612:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.940615:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025cdf lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6b3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.940617:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025c8b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.940638:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000099ea5b94/0xd5401cbabc02098e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f90 expref: 596 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.940639:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025c8b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.940658:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94360550d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+409:3626:x1870152993153408:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4247us (4533us total) trans 4294975977 rc 0/0 00010000:00010000:0.0:1783520935.940658:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b3:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.940664:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.940666:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025c8b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.940670:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025cdf lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.940674:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44474 00010000:00010000:3.0:1783520935.940679:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.940689:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025c92 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.940693:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025c8b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.940705:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000099ea5b94/0xd5401cbabc02098e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f90 expref: 596 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.940709:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.940723:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000099ea5b94/0xd5401cbabc02098e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f90 expref: 596 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.940725:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025c92 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.940740:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.940745:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.940750:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025c92 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.940750:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.940751:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503849880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993155584:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.940771:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025c92 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.940779:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025c92 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.940803:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.940804:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f000 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.940807:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.940811:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520935.940839:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94360550df80/2, rc 0, transno 4294975977 00000100:00100000:3.0:1783520935.940856:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94360550df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993153408:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.940859:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503848700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993154560:192.168.204.33@tcp:104: 00000004:00080000:3.0:1783520935.940877:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94360550dc00/2, rc 0, transno 4294975976 00010000:00010000:1.0:1783520935.940879:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025cbc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.940887:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94360550dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993153536:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.940909:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503849c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993155712:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.940927:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350384ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993155840:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.940943:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.941011:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993155712 00000100:00100000:1.0:1783520935.941016:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993155712 00000100:00100000:0.0:1783520935.941025:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.941051:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993155840 00000100:00100000:1.0:1783520935.941055:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993155840 00000100:00100000:2.0:1783520935.941078:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.941090:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44482 00000100:00100000:1.0:1783520935.941103:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44483 00000100:00100000:0.0:1783520935.941104:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d321c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+410:3626:x1870152993155712:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.941114:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.941115:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350384b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+410:3627:x1870152993155840:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.941126:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1716 count 1 00002000:00080000:1.0:1783520935.941132:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1717 count 1 00000100:00100000:2.0:1783520935.941134:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002419200 00000100:00100000:2.0:1783520935.941140:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002419200 00010000:00010000:0.0:1783520935.941141:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025ce6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6b4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.941149:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025ced lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6b5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.941156:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009cb77ccb/0xd5401cbabc020987 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f9e expref: 597 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.941162:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008395e9ef/0xd5401cbabc020964 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f58 expref: 597 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.941169:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000559467e6/0xd5401cbabc0209aa lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fac expref: 597 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.941177:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b4:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.941189:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.941192:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b5:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.941196:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025ce6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.941198:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b5:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.941203:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002419072 00010000:00010000:1.0:1783520935.941207:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025ced lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.941209:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002419072 00010000:00010000:2.0:1783520935.941228:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000098746591/0xd5401cbabc020972 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f66 expref: 598 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.941239:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009cb77ccb/0xd5401cbabc020987 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f9e expref: 598 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.941247:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000559467e6/0xd5401cbabc0209aa lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fac expref: 598 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.941255:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36068 00010000:00010000:0.0:1783520935.941256:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009cb77ccb/0xd5401cbabc020987 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f9e expref: 598 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.941269:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000559467e6/0xd5401cbabc0209aa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fac expref: 598 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.941272:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+598:2388:x1870153002419200:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.941277:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503849180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993155968:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.941290:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.941295:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94360550f100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993156096:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.941299:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008395e9ef/0xd5401cbabc020964 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6af:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f58 expref: 598 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.941315:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008395e9ef/0xd5401cbabc020964 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6af:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f58 expref: 598 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.941323:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351db63600 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.941332:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e600 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.941334:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008395e9ef/0xd5401cbabc020964 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6af:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f58 expref: 598 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.941347:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6af:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.941352:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6af:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.941379:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36069 00000100:00100000:1.0:1783520935.941391:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350805b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+597:2388:x1870153002419072:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.941401:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.941408:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000098746591/0xd5401cbabc020972 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6b0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f66 expref: 597 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.941412:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c2d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993155072:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.941428:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000098746591/0xd5401cbabc020972 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6b0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f66 expref: 597 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.941433:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025cd1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.941446:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.941451:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.941456:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025cc3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.941465:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025cc3 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6b0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.941481:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.941518:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350805b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+597:2388:x1870153002419072:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 131us (409us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.941525:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36069 00000100:00100000:3.0:1783520935.941549:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.941564:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516359180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993154816:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.941580:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002419456 00010000:00010000:1.0:1783520935.941582:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025cca lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.941587:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002419456 00010000:00010000:2.0:1783520935.941599:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025cbc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.941609:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000034269690/0xd5401cbabc020979 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f74 expref: 597 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.941614:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025cbc lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.941645:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.941645:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36070 00000100:00100000:0.0:1783520935.941658:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f2850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+597:2390:x1870153002419456:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.941662:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025cbc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.941668:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.941670:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025cbc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.941675:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+597:2388:x1870153002419200:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 405us (595us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.941678:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000034269690/0xd5401cbabc020979 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6b1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f74 expref: 597 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.941689:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025cbc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.941696:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36068 00010000:00010000:1.0:1783520935.941707:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025cbc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.941708:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000034269690/0xd5401cbabc020979 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6b1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f74 expref: 597 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.941736:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.941737:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000034269690/0xd5401cbabc020979 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6b1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f74 expref: 596 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.941750:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.941754:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.941759:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025cd1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.941768:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025cd1 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.941791:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.941814:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350384aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+411:3626:x1870152993154304:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4076us (4203us total) trans 4294975978 rc 0/0 00000100:00100000:0.0:1783520935.941820:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f2850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+595:2390:x1870153002419456:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 162us (271us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.941825:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44476 00010000:00010000:2.0:1783520935.941829:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.941837:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025cbc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.941837:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36070 00010000:00010000:2.0:1783520935.941850:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.941855:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025cbc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.941864:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025cbc lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.941870:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516358a80/2, rc 0, transno 4294975978 00010000:00010000:2.0:1783520935.941879:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025cbc lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.941884:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516358a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993154304:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.941896:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.941897:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025cbc lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.941901:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025cd1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.941918:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025cd1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.941926:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.941929:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025cd1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.941930:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.941935:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.941941:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025cd1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.941963:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.942001:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.942007:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025cd1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.942015:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.942019:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025cd1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.942028:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025cd1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.942038:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025cd1 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.942043:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503849500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993155328:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.942045:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025cd1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.942061:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351635b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+411:3627:x1870152993154944:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3224us (3298us total) trans 4294975979 rc 0/0 00000100:00100000:3.0:1783520935.942063:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.942064:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.942068:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.942068:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025cd8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.942072:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.942078:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44479 00000100:00100000:1.0:1783520935.942104:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002419328 00000100:00100000:1.0:1783520935.942109:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002419328 00010000:00010000:2.0:1783520935.942111:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000098746591/0xd5401cbabc020972 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f66 expref: 594 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.942118:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94360550d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993156224:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.942128:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002419584 00010000:00010000:1.0:1783520935.942130:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000062a21c8d/0xd5401cbabc022aff lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31ef expref: 594 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.942133:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c0000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993154688:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.942135:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002419584 00000100:00100000:0.0:1783520935.942154:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.942155:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025cc3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.942161:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000058c6dac1/0xd5401cbabc020980 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f82 expref: 595 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.942165:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025cc3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.942167:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36071 00000100:00100000:1.0:1783520935.942185:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d321880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+595:2390:x1870153002419328:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.942187:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025cc3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.942196:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025cc3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.942197:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000004:00080000:0.0:1783520935.942200:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435161c2680/2, rc 0, transno 4294975979 00010000:00010000:1.0:1783520935.942207:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000062a21c8d/0xd5401cbabc022aff lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6ae:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31ef expref: 595 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.942214:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025cc3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.942216:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993154944:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.942232:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000062a21c8d/0xd5401cbabc022aff lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6ae:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31ef expref: 595 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.942245:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36072 00010000:00010000:1.0:1783520935.942252:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000062a21c8d/0xd5401cbabc022aff lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6ae:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31ef expref: 595 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.942259:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d320700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+595:2390:x1870153002419584:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.942269:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:1.0:1783520935.942271:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ae:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.942273:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.942278:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ae:0x0].0x0 since=0s v=(1 0) 00000100:00100000:0.0:1783520935.942280:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503849500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993156352:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.942283:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000058c6dac1/0xd5401cbabc020980 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6b2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f82 expref: 594 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.942289:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025cca lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.942305:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025cca lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.942307:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000058c6dac1/0xd5401cbabc020980 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6b2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f82 expref: 594 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.942315:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.942338:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000058c6dac1/0xd5401cbabc020980 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6b2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f82 expref: 594 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.942345:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.942352:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b2:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.942355:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993156224 00010000:00010000:2.0:1783520935.942385:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b2:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.942387:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993156224 00010000:00010000:2.0:1783520935.942390:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025cd8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.942400:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025cd8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.942408:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d321880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+593:2390:x1870153002419328:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 226us (514us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.942415:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993156352 00000100:00100000:1.0:1783520935.942420:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36071 00000100:00100000:0.0:1783520935.942421:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993156352 00010000:00010000:2.0:1783520935.942426:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.942454:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025cd8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.942456:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d320700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+592:2390:x1870153002419584:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 199us (394us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.942456:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44484 00000100:00100000:0.0:1783520935.942470:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503849f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+412:3628:x1870152993156224:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.942476:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025cd8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.942477:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36072 00010000:00010000:1.0:1783520935.942489:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025cd8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.942490:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1718 count 1 00010000:00010000:1.0:1783520935.942512:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025cd8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.942517:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025cf4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6b6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.942524:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44485 00010000:00010000:1.0:1783520935.942531:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520935.942538:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503848700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+412:3626:x1870152993156352:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.942539:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f6bcb380/0xd5401cbabc0229ee lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f30de expref: 591 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.942551:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x6b6:0x0].0x0 since=36s v=(0 0) 00002000:00080000:3.0:1783520935.942555:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1719 count 1 00010000:00010000:0.0:1783520935.942555:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.942564:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025cf4 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.942592:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f6bcb380/0xd5401cbabc0229ee lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30de expref: 591 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.942603:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.942610:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f6bcb380/0xd5401cbabc0229ee lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30de expref: 591 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.942625:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350384bb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993156480:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.942626:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002419712 00000100:00100000:2.0:1783520935.942632:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002419712 00000100:00100000:1.0:1783520935.942634:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516359f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+413:3628:x1870152993155200:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3013us (3081us total) trans 4294975980 rc 0/0 00000100:00100000:1.0:1783520935.942643:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44480 00010000:00010000:2.0:1783520935.942656:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000099ea5b94/0xd5401cbabc02098e lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f90 expref: 592 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.942658:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350384b800/2, rc 0, transno 4294975980 00010000:00010000:1.0:1783520935.942665:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.942674:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025cd8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.942674:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350384b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993155200:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.942691:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36073 00010000:00010000:1.0:1783520935.942695:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.942700:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025cd8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.942707:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508059f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+592:2388:x1870153002419712:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.942709:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025cfb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6b7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.942714:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025cd8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.942721:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.942728:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000099ea5b94/0xd5401cbabc02098e lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6b3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f90 expref: 592 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.942729:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025cd8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.942738:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993156608:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.942759:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025cd8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.942776:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000099ea5b94/0xd5401cbabc02098e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6b3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f90 expref: 592 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.942777:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.942793:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.942797:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001383fe32/0xd5401cbabc02099c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fc8 expref: 592 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.942797:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.942801:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.942803:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025cdf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.942806:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.942812:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.942824:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b7:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.942826:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025cdf lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6b3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.942830:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993156608 00010000:00010000:3.0:1783520935.942831:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b7:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.942838:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993156608 00010000:00010000:3.0:1783520935.942841:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025cfb lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.942845:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607eb8a80 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.942852:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.942876:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508059f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+592:2388:x1870153002419712:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 176us (278us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.942882:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44486 00010000:00010000:0.0:1783520935.942883:0:35276:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000099ea5b94/0xd5401cbabc02098e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f90 expref: 592 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.942888:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36073 00000100:00100000:1.0:1783520935.942897:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+413:3626:x1870152993156608:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.942906:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503849880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993155584:192.168.204.33@tcp:104: 00002000:00080000:1.0:1783520935.942916:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1720 count 1 00010000:00010000:0.0:1783520935.942929:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025cdf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.942931:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025d02 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6b8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.942948:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001383fe32/0xd5401cbabc02099c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fc8 expref: 590 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.942960:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000000ab7be62/0xd5401cbabc0209a3 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fd6 expref: 590 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.942975:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503849180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993155968:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.942978:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001383fe32/0xd5401cbabc02099c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fc8 expref: 590 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.942979:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b8:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.942989:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b8:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.942993:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.942993:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025ce6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.942996:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025d02 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.943026:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002419840 00000100:00100000:0.0:1783520935.943031:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002419840 00010000:00010000:1.0:1783520935.943036:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000000ab7be62/0xd5401cbabc0209a3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fd6 expref: 591 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.943048:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d321f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993156736:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.943050:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009cb77ccb/0xd5401cbabc020987 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f9e expref: 591 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.943052:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000000ab7be62/0xd5401cbabc0209a3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fd6 expref: 591 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.943080:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516359180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993156864:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.943092:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8840 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.943092:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36074 00000100:00100000:0.0:1783520935.943105:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350805ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+591:2388:x1870153002419840:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.943116:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.943119:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e240 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.943123:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009cb77ccb/0xd5401cbabc020987 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6b4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f9e expref: 591 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.943142:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009cb77ccb/0xd5401cbabc020987 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6b4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f9e expref: 591 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.943152:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94360550f100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993156096:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.943160:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009cb77ccb/0xd5401cbabc020987 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6b4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0f9e expref: 591 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.943166:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.943175:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.943176:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025ced lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.943182:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.943193:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025ce6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.943202:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025ce6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.943208:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.943208:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025cca lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.943216:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025cc3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.943222:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025cca lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.943223:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.943232:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.943239:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025cc3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.943246:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025cca lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.943253:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025cc3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.943258:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350805ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+590:2388:x1870153002419840:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 155us (268us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.943261:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+414:3625:x1870152993154432:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5325us (5409us total) trans 4294975981 rc 0/0 00010000:00010000:1.0:1783520935.943261:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025cca lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.943277:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36074 00000100:00100000:2.0:1783520935.943281:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44477 00010000:00010000:1.0:1783520935.943299:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.943308:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.943312:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025cc3 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.943314:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025cca lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.943317:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435161c3100/2, rc 0, transno 4294975981 00000100:00100000:0.0:1783520935.943331:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993154432:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.943334:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025cc3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.943335:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.943344:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025cca lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.943355:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025cca lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.943391:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.943394:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025cca lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.943395:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.943400:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.943407:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025cca lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.943416:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1783520935.943428:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993156992:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.943429:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.943433:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.943437:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.943438:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516358700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+414:3628:x1870152993154176:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5145us (6155us total) trans 4294975982 rc 0/0 00000100:00100000:1.0:1783520935.943448:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44478 00000100:00100000:0.0:1783520935.943465:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.943481:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993156992 00000100:00100000:1.0:1783520935.943486:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993156992 00000004:00080000:0.0:1783520935.943507:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516359880/2, rc 0, transno 4294975982 00000100:00100000:0.0:1783520935.943520:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516359880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993154176:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.943536:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44487 00000100:00100000:1.0:1783520935.943549:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503849180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+414:3628:x1870152993156992:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.943556:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025ce6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.943564:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1721 count 1 00010000:00010000:0.0:1783520935.943568:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025ce6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.943578:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d09 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6b9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.943580:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025ce6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.943604:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025ce6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.943607:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007800d8f4/0xd5401cbabc0209b1 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fe4 expref: 589 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.943624:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b9:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.943627:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b9:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.943629:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350805b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993157120:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.943633:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d09 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.943658:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.943665:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007800d8f4/0xd5401cbabc0209b1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fe4 expref: 589 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.943678:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007800d8f4/0xd5401cbabc0209b1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fe4 expref: 589 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.943687:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993157120 00000100:00100000:1.0:1783520935.943691:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94360550f100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993157248:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.943693:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993157120 00010000:00010000:0.0:1783520935.943708:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.943719:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.943726:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025ce6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.943732:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44488 00010000:00010000:3.0:1783520935.943741:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb000 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520935.943745:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503849880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+415:3628:x1870152993157120:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.943747:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025ce6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.943747:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.943760:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025ce6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.943767:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1722 count 1 00010000:00010000:3.0:1783520935.943769:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025ce6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.943775:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025ce6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.943781:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025d10 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6ba:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.943789:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.943792:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.943793:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000abca14bd/0xd5401cbabc0209bf lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ff2 expref: 589 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.943796:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.943801:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ba:0x0].0x0 since=37s v=(0 0) 00000100:00100000:0.0:1783520935.943807:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d321c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+416:3626:x1870152993155712:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2706us (2862us total) trans 4294975983 rc 0/0 00010000:00010000:2.0:1783520935.943808:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ba:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.943818:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025d10 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.943822:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44482 00000004:00080000:1.0:1783520935.943823:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503849c00/2, rc 0, transno 4294975983 00000100:00100000:1.0:1783520935.943841:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503849c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993155712:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.943851:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000abca14bd/0xd5401cbabc0209bf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ff2 expref: 589 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.943851:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025cdf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.943877:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025cdf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.943880:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000abca14bd/0xd5401cbabc0209bf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ff2 expref: 589 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.943891:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993157504:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.943896:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025cdf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.943910:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514f98e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993157376:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.943910:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025cdf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.943932:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.943936:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520935.943944:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435126766c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.943948:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.943954:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025cdf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.943962:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbebe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.943966:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025cdf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.943976:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025cdf lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.943980:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993157504 00000100:00100000:1.0:1783520935.943986:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993157504 00010000:00010000:0.0:1783520935.943992:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025cdf lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.944002:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025cdf lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.944016:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44489 00010000:00010000:0.0:1783520935.944024:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.944027:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436342b9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+416:3625:x1870152993157504:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.944028:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.944032:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00002000:00080000:1.0:1783520935.944042:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1723 count 1 00010000:00010000:1.0:1783520935.944054:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025d17 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6bb:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.944067:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007487c67c/0xd5401cbabc0209c6 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1000 expref: 589 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.944074:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6bb:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.944078:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6bb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.944082:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025d17 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.944104:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007487c67c/0xd5401cbabc0209c6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1000 expref: 589 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.944116:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007487c67c/0xd5401cbabc0209c6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1000 expref: 589 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.944117:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+417:3625:x1870152993155456:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3528us (3588us total) trans 4294975984 rc 0/0 00000100:00100000:0.0:1783520935.944135:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44481 00000100:00100000:1.0:1783520935.944142:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9436342b8a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993157632:192.168.204.33@tcp:104: 00000004:00080000:0.0:1783520935.944172:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351635aa00/2, rc 0, transno 4294975984 00000100:00100000:1.0:1783520935.944183:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.944185:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351635aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993155456:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.944199:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.944224:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002419968 00000100:00100000:1.0:1783520935.944229:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002419968 00000100:00100000:0.0:1783520935.944244:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:1.0:1783520935.944249:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000559467e6/0xd5401cbabc0209aa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fac expref: 590 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.944257:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993157760:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.944280:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36075 00000100:00100000:1.0:1783520935.944293:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d322300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+590:2389:x1870153002419968:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.944294:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.944306:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.944312:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000559467e6/0xd5401cbabc0209aa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6b5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fac expref: 590 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.944324:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000559467e6/0xd5401cbabc0209aa lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6b5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fac expref: 590 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.944329:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993157760 00000100:00100000:0.0:1783520935.944335:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993157760 00010000:00010000:1.0:1783520935.944349:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000559467e6/0xd5401cbabc0209aa lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6b5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fac expref: 590 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.944382:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.944386:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.944391:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025ced lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.944392:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44490 00010000:00010000:1.0:1783520935.944404:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025ced lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.944405:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+417:3625:x1870152993157760:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.944426:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1724 count 1 00010000:00010000:1.0:1783520935.944437:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.944441:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025d1e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6bc:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.944447:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025ced lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.944457:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025ced lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.944457:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000199892ee/0xd5401cbabc0209cd lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f100e expref: 589 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.944470:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d322300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+589:2389:x1870153002419968:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 179us (271us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.944481:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025ced lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.944484:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6bc:0x0].0x0 since=37s v=(0 0) 00000100:00100000:1.0:1783520935.944490:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36075 00010000:00010000:2.0:1783520935.944507:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025ced lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.944507:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6bc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.944519:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025d1e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.944534:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.944541:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.944544:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000199892ee/0xd5401cbabc0209cd lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f100e expref: 588 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.944548:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025ced lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.944565:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000199892ee/0xd5401cbabc0209cd lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f100e expref: 588 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.944566:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.944573:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025ced lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.944585:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11c700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993157888:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.944588:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025ced lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.944603:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025ced lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.944633:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350384b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+418:3627:x1870152993155840:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3520us (3609us total) trans 4294975985 rc 0/0 00000100:00100000:0.0:1783520935.944639:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351db63600 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.944644:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44483 00000004:00080000:1.0:1783520935.944652:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350384ad80/2, rc 0, transno 4294975985 00000100:00100000:1.0:1783520935.944664:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350384ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993155840:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.944706:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025ced lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.944707:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1783520935.944718:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993158016:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.944728:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.944732:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.944735:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.944749:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.944775:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993158016 00000100:00100000:3.0:1783520935.944780:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993158016 00000100:00100000:1.0:1783520935.944806:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350384bb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993156480:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.944814:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44491 00000100:00100000:3.0:1783520935.944824:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+418:3628:x1870152993158016:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.944827:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025cf4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.944830:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d321f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993156736:192.168.204.33@tcp:104: 00002000:00080000:3.0:1783520935.944839:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1725 count 1 00010000:00010000:0.0:1783520935.944854:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025cfb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.944856:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025d25 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6bd:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.944869:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516359180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993156864:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.944885:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006fce27d6/0xd5401cbabc0209d4 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f101c expref: 588 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.944890:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025d02 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.944896:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6bd:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.944904:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6bd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.944910:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025d25 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.944929:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006fce27d6/0xd5401cbabc0209d4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f101c expref: 588 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.944938:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006fce27d6/0xd5401cbabc0209d4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f101c expref: 588 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.944987:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d321180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993158144:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.945017:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d83c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.946272:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.946308:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.946329:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002420352 00000100:00100000:2.0:1783520935.946336:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002420352 00000100:00100000:3.0:1783520935.946340:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002420096 00000100:00100000:1.0:1783520935.946341:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.946345:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002420096 00010000:00010000:2.0:1783520935.946390:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000000ab7be62/0xd5401cbabc0209a3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fd6 expref: 590 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.946398:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f6bcb380/0xd5401cbabc0229ee lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30de expref: 590 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.946424:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002420224 00000100:00100000:2.0:1783520935.946428:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002420224 00000100:00100000:3.0:1783520935.946430:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36076 00000100:00100000:3.0:1783520935.946440:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436342b8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+591:2388:x1870153002420352:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.946443:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001383fe32/0xd5401cbabc02099c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fc8 expref: 591 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.946447:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36077 00010000:00010000:3.0:1783520935.946453:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.946458:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.946463:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000000ab7be62/0xd5401cbabc0209a3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6b8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fd6 expref: 591 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.946465:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f1050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+591:2388:x1870153002420096:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.946478:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002420480 00000100:00100000:2.0:1783520935.946481:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002420480 00010000:00010000:3.0:1783520935.946482:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000000ab7be62/0xd5401cbabc0209a3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6b8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fd6 expref: 591 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.946505:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.946507:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36078 00010000:00010000:2.0:1783520935.946509:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007800d8f4/0xd5401cbabc0209b1 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fe4 expref: 592 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.946513:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000ab7be62/0xd5401cbabc0209a3 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6b8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fd6 expref: 592 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.946516:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f6bcb380/0xd5401cbabc0229ee lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6b6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30de expref: 592 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.946524:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436342bad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+592:2389:x1870153002420224:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.946535:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36079 00010000:00010000:3.0:1783520935.946536:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.946540:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.946542:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.946547:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025d02 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.946547:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f6bcb380/0xd5401cbabc0229ee lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6b6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30de expref: 591 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.946552:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001383fe32/0xd5401cbabc02099c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6b7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fc8 expref: 591 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.946553:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436342baa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+591:2389:x1870153002420480:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.946571:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025d02 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.946571:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.946578:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007800d8f4/0xd5401cbabc0209b1 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6b9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fe4 expref: 591 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.946586:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f6bcb380/0xd5401cbabc0229ee lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6b6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30de expref: 591 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.946590:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001383fe32/0xd5401cbabc02099c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6b7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fc8 expref: 591 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.946596:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007800d8f4/0xd5401cbabc0209b1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6b9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fe4 expref: 591 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.946609:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.946610:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.946616:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.946620:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001383fe32/0xd5401cbabc02099c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6b7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fc8 expref: 590 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.946621:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.946623:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025cf4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.946631:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.946642:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d09 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.946646:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025cf4 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.946647:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436342b8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+589:2388:x1870153002420352:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 207us (376us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.946648:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.946658:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6b7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.946661:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d09 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6b9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.946664:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36076 00010000:00010000:1.0:1783520935.946670:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025cfb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.946684:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025cfb lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.946689:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.946690:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.946715:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.946720:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f1050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+588:2388:x1870153002420096:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 259us (408us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.946726:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436342baa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+588:2389:x1870153002420480:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 178us (268us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.946733:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025cfb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.946734:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36077 00000100:00100000:1.0:1783520935.946743:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436342bad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+588:2389:x1870153002420224:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 223us (401us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.946748:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025cfb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.946748:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36079 00000100:00100000:1.0:1783520935.946760:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36078 00010000:00010000:3.0:1783520935.946761:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025cfb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.946771:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025cfb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.946791:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520935.946809:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514f98e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993157376:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.946809:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025d02 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.946816:0:9151:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007800d8f4/0xd5401cbabc0209b1 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0fe4 expref: 585 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.946831:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025d10 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.946837:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025d02 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.946843:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94360550f100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993157248:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.946858:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025d02 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.946859:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.946868:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025cfb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.946870:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025d02 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.946877:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d09 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.946890:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.946898:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025cfb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.946900:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d09 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.946907:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.946925:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025cfb lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.946925:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d09 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.946927:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503848700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+419:3626:x1870152993156352:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4393us (4617us total) trans 4294975986 rc 0/0 00000004:00080000:1.0:1783520935.946942:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503849500/2, rc 0, transno 4294975986 00000100:00100000:3.0:1783520935.946943:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44485 00010000:00010000:2.0:1783520935.946950:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025cfb lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.946953:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d09 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.946964:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520935.946965:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503849500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993156352:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.946968:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025cfb lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.946971:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025d02 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.946991:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.946998:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025d02 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.946998:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d09 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.947003:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025cf4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.947012:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.947014:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.947017:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.947022:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025d02 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.947025:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025cf4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.947045:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025d02 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.947049:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025cf4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.947063:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025d02 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.947066:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025cf4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.947091:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.947095:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to regular blp list, 0 blwis in pool 00010000:00010000:3.0:1783520935.947097:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025cf4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.947107:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.947111:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025cf4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.947126:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025cf4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.947133:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.947139:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025cf4 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.947149:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025cf4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.947150:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993158272:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.947188:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503849f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+419:3628:x1870152993156224:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4719us (5034us total) trans 4294975987 rc 0/0 00000100:00100000:0.0:1783520935.947191:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.947203:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44484 00010000:00010000:3.0:1783520935.947221:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.947225:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.947226:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993158272 00010000:00010000:3.0:1783520935.947228:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.947231:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993158272 00000004:00080000:0.0:1783520935.947235:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94360550d880/2, rc 0, transno 4294975987 00000100:00100000:0.0:1783520935.947247:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94360550d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993156224:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.947272:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44492 00000100:00100000:1.0:1783520935.947282:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+419:3628:x1870152993158272:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.947286:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.947291:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.947298:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d09 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.947302:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:3.0:1783520935.947310:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.947315:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d09 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.947316:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514f98e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993158400:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.947331:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d09 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.947343:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d09 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.947354:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.947401:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d09 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.947422:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.947424:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.947426:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.947431:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993158400 00000100:00100000:2.0:1783520935.947435:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993158400 00000100:00100000:0.0:1783520935.947438:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503849180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+420:3628:x1870152993156992:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3892us (3973us total) trans 4294975988 rc 0/0 00000100:00100000:0.0:1783520935.947447:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44487 00000100:00100000:2.0:1783520935.947469:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44493 00000100:00100000:2.0:1783520935.947480:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436342ba300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+420:3626:x1870152993158400:12345-0@lo:6:osp-syn-1-0.0 00000004:00080000:0.0:1783520935.947487:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435161c2d80/2, rc 0, transno 4294975988 00002000:00080000:2.0:1783520935.947511:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1727 count 1 00000100:00100000:0.0:1783520935.947511:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993156992:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.947531:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025d2c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6bf:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.947545:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+421:3626:x1870152993156608:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4651us (4771us total) trans 4294975989 rc 0/0 00010000:00010000:2.0:1783520935.947546:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c0ce98ea/0xd5401cbabc0209db lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1038 expref: 584 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.947553:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:2.0:1783520935.947565:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6bf:0x0].0x0 since=37s v=(0 0) 00000100:00100000:1.0:1783520935.947565:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44486 00000100:00100000:0.0:1783520935.947567:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d321880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993158528:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.947573:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6bf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.947579:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025d2c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.947602:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435161c0000/2, rc 0, transno 4294975989 00010000:00010000:2.0:1783520935.947604:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c0ce98ea/0xd5401cbabc0209db lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1038 expref: 584 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.947619:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993156608:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.947620:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.947621:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c0ce98ea/0xd5401cbabc0209db lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1038 expref: 584 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.947638:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435055cea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993158656:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.947643:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993158528 00000100:00100000:1.0:1783520935.947649:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993158528 00000100:00100000:1.0:1783520935.947683:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44494 00000100:00100000:0.0:1783520935.947690:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993158784:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.947692:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435126766c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.947695:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635785850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+421:3625:x1870152993158528:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.947711:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1728 count 1 00010000:00010000:1.0:1783520935.947725:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025d33 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6c0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.947727:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.947741:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007caaaca1/0xd5401cbabc0209e9 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1046 expref: 584 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.947748:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c0:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.947752:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.947756:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025d33 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.947773:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993158784 00000100:00100000:0.0:1783520935.947778:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993158784 00010000:00010000:1.0:1783520935.947779:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007caaaca1/0xd5401cbabc0209e9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1046 expref: 584 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.947791:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007caaaca1/0xd5401cbabc0209e9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1046 expref: 584 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.947802:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9436342b9500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993158912:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.947810:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44495 00000100:00100000:0.0:1783520935.947820:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+422:3627:x1870152993158784:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.947834:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1729 count 1 00000100:00100000:2.0:1783520935.947840:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.947846:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e3c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.947850:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025d3a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6c1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.947861:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002420608 00010000:00010000:0.0:1783520935.947864:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000078cbc7e7/0xd5401cbabc0209fe lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1054 expref: 584 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.947867:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002420608 00002000:00080000:1.0:1783520935.947869:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1726 count 1 00010000:00010000:0.0:1783520935.947877:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c1:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.947881:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.947887:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025d41 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6be:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.947888:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025d3a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.947889:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000abca14bd/0xd5401cbabc0209bf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ff2 expref: 585 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.947921:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000289a3009/0xd5401cbabc0228dd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fcd expref: 585 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.947929:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000078cbc7e7/0xd5401cbabc0209fe lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1054 expref: 585 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.947933:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x6be:0x0].0x0 since=36s v=(0 0) 00000100:00100000:2.0:1783520935.947934:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36080 00010000:00010000:1.0:1783520935.947942:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6be:0x0].0x0 since=0s v=(1 0) 00000100:00100000:2.0:1783520935.947949:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435055cdf80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+585:2389:x1870153002420608:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.947950:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000078cbc7e7/0xd5401cbabc0209fe lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1054 expref: 585 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.947952:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025d41 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.947963:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.947972:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000abca14bd/0xd5401cbabc0209bf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ba:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ff2 expref: 585 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.947983:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11c380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993159040:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.947991:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000abca14bd/0xd5401cbabc0209bf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6ba:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ff2 expref: 585 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.948000:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000289a3009/0xd5401cbabc0228dd lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fcd expref: 585 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.948011:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000abca14bd/0xd5401cbabc0209bf lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6ba:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f0ff2 expref: 585 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.948016:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000289a3009/0xd5401cbabc0228dd lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fcd expref: 585 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.948017:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.948023:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943515746cc0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.948028:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ba:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.948034:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ba:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.948040:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9436342b8000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993159168:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.948041:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025d10 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.948056:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11c700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993157888:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.948059:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.948059:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025d10 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.948075:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e240 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.948081:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025d1e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.948082:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.948105:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002420736 00000100:00100000:2.0:1783520935.948110:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435055cdf80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+584:2389:x1870153002420608:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 162us (270us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.948111:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9436342b8a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993157632:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.948111:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002420736 00000100:00100000:2.0:1783520935.948126:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36080 00010000:00010000:0.0:1783520935.948131:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000199892ee/0xd5401cbabc0209cd lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f100e expref: 585 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.948135:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025d17 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.948142:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002420864 00000100:00100000:2.0:1783520935.948148:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002420864 00010000:00010000:2.0:1783520935.948167:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007487c67c/0xd5401cbabc0209c6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1000 expref: 585 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.948167:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36081 00000100:00100000:3.0:1783520935.948174:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.948183:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d322300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+585:2390:x1870153002420736:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.948185:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002420992 00010000:00010000:0.0:1783520935.948193:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.948197:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002420992 00000100:00100000:2.0:1783520935.948201:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36082 00010000:00010000:0.0:1783520935.948201:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000199892ee/0xd5401cbabc0209cd lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6bc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f100e expref: 585 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.948216:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d321f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+586:2389:x1870153002420864:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.948218:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000199892ee/0xd5401cbabc0209cd lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6bc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f100e expref: 586 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.948222:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006fce27d6/0xd5401cbabc0209d4 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f101c expref: 586 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.948234:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.948244:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007487c67c/0xd5401cbabc0209c6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6bb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1000 expref: 586 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.948251:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000199892ee/0xd5401cbabc0209cd lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6bc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f100e expref: 586 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.948259:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36083 00010000:00010000:2.0:1783520935.948265:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007487c67c/0xd5401cbabc0209c6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6bb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1000 expref: 586 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.948272:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6bc:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.948276:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a80380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+585:2390:x1870153002420992:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.948281:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6bc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.948288:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007487c67c/0xd5401cbabc0209c6 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6bb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1000 expref: 585 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.948288:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025d1e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.948295:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d321180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993158144:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.948301:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.948311:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025d25 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6bd:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.948314:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006fce27d6/0xd5401cbabc0209d4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6bd:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f101c expref: 585 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.948316:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025d1e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.948339:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6bb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.948344:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6bb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.948347:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006fce27d6/0xd5401cbabc0209d4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6bd:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f101c expref: 584 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.948350:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025d17 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.948354:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.948396:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025d17 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.948403:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d322300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+584:2390:x1870153002420736:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 222us (384us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.948407:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006fce27d6/0xd5401cbabc0209d4 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6bd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f101c expref: 584 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.948422:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36081 00010000:00010000:2.0:1783520935.948427:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.948429:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6bd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.948435:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6bd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.948440:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025d25 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.948449:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025d1e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.948450:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025d25 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.948455:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d321f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+582:2389:x1870153002420864:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 240us (394us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.948475:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36082 00010000:00010000:0.0:1783520935.948475:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025d1e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.948487:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025d1e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.948506:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.948509:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025d1e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.948513:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025d10 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.948529:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025d10 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.948537:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a80380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+581:2390:x1870153002420992:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 266us (363us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.948537:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.948541:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025d10 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.948557:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025d10 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.948557:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36083 00010000:00010000:2.0:1783520935.948574:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.948579:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025d1e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.948587:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.948590:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025d17 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.948592:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025d1e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.948614:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025d1e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.948616:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025d17 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.948630:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+423:3625:x1870152993157760:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4224us (4336us total) trans 4294975990 rc 0/0 00010000:00010000:2.0:1783520935.948639:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025d1e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.948646:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025d17 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.948648:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44490 00010000:00010000:2.0:1783520935.948659:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025d1e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.948663:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025d17 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.948680:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e11ea00/2, rc 0, transno 4294975990 00010000:00010000:3.0:1783520935.948685:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025d25 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.948688:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.948690:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520935.948696:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993157760:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.948697:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025d17 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.948699:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025d25 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.948710:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbae00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.948712:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025d25 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.948713:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025d17 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.948717:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.948723:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025d25 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.948724:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025d17 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.948741:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025d17 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.948746:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.948749:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025d17 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.948757:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520935.948759:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:3.0:1783520935.948761:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025d25 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.948764:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.948766:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.948769:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.948769:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.948771:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993159296:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.948773:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025d25 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.948782:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.948783:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436342b9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+423:3625:x1870152993157504:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4756us (4849us total) trans 4294975991 rc 0/0 00010000:00010000:3.0:1783520935.948786:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025d25 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.948800:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44489 00000100:00100000:0.0:1783520935.948807:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.948808:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025d25 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.948817:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025d25 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.948827:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993159296 00000100:00100000:1.0:1783520935.948832:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993159296 00010000:00010000:3.0:1783520935.948836:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.948840:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025d10 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.948848:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185000 already unused, calling callback (ffffffffc0bf9e90) 00000004:00080000:0.0:1783520935.948854:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363d320000/2, rc 0, transno 4294975991 00000100:00100000:0.0:1783520935.948866:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993157504:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.948868:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44496 00000100:00100000:2.0:1783520935.948880:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503849880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+424:3628:x1870152993157120:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5137us (5219us total) trans 4294975992 rc 0/0 00000100:00100000:1.0:1783520935.948884:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+424:3627:x1870152993159296:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.948889:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025d10 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.948893:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44488 00002000:00080000:1.0:1783520935.948903:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1730 count 1 00010000:00010000:3.0:1783520935.948905:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025d10 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.948922:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025d10 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.948928:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:3.0:1783520935.948929:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc025d10 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.948931:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350805b100/2, rc 0, transno 4294975992 00000100:00100000:1.0:1783520935.948940:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993159424:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.948945:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350805b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993157120:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.948954:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.948958:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.948960:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.948968:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.948981:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.948988:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1783520935.948999:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993159552:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.949009:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993159424 00000100:00100000:2.0:1783520935.949014:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993159424 00010000:00010000:1.0:1783520935.949024:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025d48 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6c2:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.949032:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.949044:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000cdccf04a/0xd5401cbabc0209f0 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1062 expref: 580 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.949046:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993159552 00000100:00100000:2.0:1783520935.949052:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993159552 00010000:00010000:1.0:1783520935.949057:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c2:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.949062:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.949067:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025d48 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.949074:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44497 00000100:00100000:2.0:1783520935.949083:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44498 00000100:00100000:0.0:1783520935.949088:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436342b8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+426:3626:x1870152993159424:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.949090:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+426:3628:x1870152993158016:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4268us (4341us total) trans 4294975993 rc 0/0 00000100:00100000:2.0:1783520935.949094:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+426:3625:x1870152993159552:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.949098:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000cdccf04a/0xd5401cbabc0209f0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1062 expref: 580 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.949103:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44491 00002000:00080000:0.0:1783520935.949104:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1731 count 1 00002000:00080000:2.0:1783520935.949110:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1732 count 1 00010000:00010000:1.0:1783520935.949118:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000cdccf04a/0xd5401cbabc0209f0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1062 expref: 580 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.949122:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025d4f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6c3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.949126:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025d56 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6c4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.949139:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516359180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993159680:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.949154:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000061e81148/0xd5401cbabc0209f7 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1070 expref: 580 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.949157:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000007f0d157/0xd5401cbabc020a0c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f107e expref: 580 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.949175:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c3:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.949182:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c3:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.949191:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.949192:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c4:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.949193:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025d4f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.949202:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.949211:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025d56 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.949224:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000061e81148/0xd5401cbabc0209f7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1070 expref: 580 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.949240:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000007f0d157/0xd5401cbabc020a0c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f107e expref: 580 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.949241:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000061e81148/0xd5401cbabc0209f7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1070 expref: 580 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.949271:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000007f0d157/0xd5401cbabc020a0c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f107e expref: 580 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.949271:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11f800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993159808:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.949289:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435055cc700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993159936:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.949304:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351db63600 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.949323:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676840 going to sleep for 11 seconds 00000004:00080000:0.0:1783520935.949332:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e11df80/2, rc 0, transno 4294975993 00000100:00100000:0.0:1783520935.949349:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993158016:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.949439:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350384bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993160064:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.949464:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.949556:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993160064 00000100:00100000:3.0:1783520935.949559:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993160064 00000100:00100000:3.0:1783520935.949581:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44499 00000100:00100000:3.0:1783520935.949642:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94360550f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+426:3627:x1870152993160064:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520935.949656:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1733 count 1 00010000:00010000:3.0:1783520935.949668:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025d5d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6c5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.949680:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006eae17b0/0xd5401cbabc020a05 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f108c expref: 580 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.949693:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c5:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.949696:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.949701:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025d5d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.949708:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9436342b9500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993158912:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.949722:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006eae17b0/0xd5401cbabc020a05 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f108c expref: 580 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.949726:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025d33 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.949733:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006eae17b0/0xd5401cbabc020a05 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f108c expref: 580 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.949742:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.949750:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435055cea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993158656:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.949751:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a80e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993160192:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.949763:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025d2c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.949792:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d83c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.949803:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002421120 00000100:00100000:0.0:1783520935.949809:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002421120 00010000:00010000:0.0:1783520935.949831:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007caaaca1/0xd5401cbabc0209e9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1046 expref: 581 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.949861:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36084 00000100:00100000:0.0:1783520935.949868:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943603d94850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+581:2389:x1870153002421120:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.949877:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.949883:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007caaaca1/0xd5401cbabc0209e9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1046 expref: 581 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.949893:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007caaaca1/0xd5401cbabc0209e9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6c0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1046 expref: 581 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.949909:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007caaaca1/0xd5401cbabc0209e9 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6c0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1046 expref: 581 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.949925:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.949929:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.949934:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025d33 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.949948:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025d33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.949957:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.949983:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002421248 00010000:00010000:0.0:1783520935.949985:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.949987:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002421248 00000100:00100000:3.0:1783520935.949999:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.950007:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c0ce98ea/0xd5401cbabc0209db lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1038 expref: 581 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.950030:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9436342b8000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993159168:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.950031:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002421376 00000100:00100000:2.0:1783520935.950036:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002421376 00000100:00100000:0.0:1783520935.950036:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943603d94850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+581:2389:x1870153002421120:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 164us (298us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.950048:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025d41 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.950048:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36084 00010000:00010000:2.0:1783520935.950055:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000078cbc7e7/0xd5401cbabc0209fe lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1054 expref: 582 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.950086:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36085 00000100:00100000:1.0:1783520935.950097:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36086 00000100:00100000:2.0:1783520935.950098:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a83100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+581:2389:x1870153002421248:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.950112:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.950113:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a81500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+581:2387:x1870153002421376:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.950120:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c0ce98ea/0xd5401cbabc0209db lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6bf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1038 expref: 581 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.950126:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.950136:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11c380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993159040:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.950137:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.950138:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c0ce98ea/0xd5401cbabc0209db lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6bf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1038 expref: 581 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.950139:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000078cbc7e7/0xd5401cbabc0209fe lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1054 expref: 581 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.950161:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025d3a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6c1:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.950170:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000078cbc7e7/0xd5401cbabc0209fe lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6c1:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1054 expref: 581 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.950175:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c0ce98ea/0xd5401cbabc0209db lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6bf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1038 expref: 581 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.950202:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6bf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.950206:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6bf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.950207:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000078cbc7e7/0xd5401cbabc0209fe lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6c1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1054 expref: 580 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.950216:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025d2c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.950221:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.950228:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.950232:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025d2c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.950233:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025d3a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.950254:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025d3a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.950267:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.950277:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.950335:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11f800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993159808:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.950336:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.950343:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a83100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+579:2389:x1870153002421248:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 247us (386us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.950348:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a81500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+579:2387:x1870153002421376:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 237us (345us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.950353:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025d4f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.950380:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36085 00010000:00010000:0.0:1783520935.950395:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025d3a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.950400:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002421504 00000100:00100000:1.0:1783520935.950401:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36086 00000100:00100000:2.0:1783520935.950406:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002421504 00010000:00010000:0.0:1783520935.950409:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025d3a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.950420:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002421632 00010000:00010000:0.0:1783520935.950421:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025d3a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.950426:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000289a3009/0xd5401cbabc0228dd lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fcd expref: 578 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.950427:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002421632 00000100:00100000:3.0:1783520935.950434:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.950450:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000cdccf04a/0xd5401cbabc0209f0 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1062 expref: 579 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.950455:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025d3a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.950457:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002421760 00000100:00100000:2.0:1783520935.950463:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002421760 00010000:00010000:0.0:1783520935.950482:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.950483:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000061e81148/0xd5401cbabc0209f7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1070 expref: 580 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.950484:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36087 00010000:00010000:0.0:1783520935.950507:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520935.950512:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a83480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+580:2389:x1870153002421504:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.950516:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025d3a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.950521:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520935.950528:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36088 00010000:00010000:0.0:1783520935.950531:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbebe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.950535:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000289a3009/0xd5401cbabc0228dd lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6be:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fcd expref: 580 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.950537:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025d3a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.950543:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a80380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+580:2388:x1870153002421632:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.950558:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.950564:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025d3a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.950566:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000cdccf04a/0xd5401cbabc0209f0 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6c2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1062 expref: 580 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.950568:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000289a3009/0xd5401cbabc0228dd lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6be:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fcd expref: 580 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.950598:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025d3a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.950601:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000cdccf04a/0xd5401cbabc0209f0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1062 expref: 580 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.950606:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000289a3009/0xd5401cbabc0228dd lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6be:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fcd expref: 580 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.950623:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025d3a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.950637:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6be:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.950638:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.950643:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.950648:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.950649:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.950649:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6be:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.950651:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.950662:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025d48 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.950662:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025d41 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.950684:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36089 00010000:00010000:2.0:1783520935.950685:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025d48 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6c2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.950687:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025d41 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.950700:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d322300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+579:2387:x1870153002421760:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.950715:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.950717:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.950721:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000061e81148/0xd5401cbabc0209f7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1070 expref: 579 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.950729:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.950733:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000061e81148/0xd5401cbabc0209f7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6c3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1070 expref: 579 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.950745:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000061e81148/0xd5401cbabc0209f7 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6c3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1070 expref: 579 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.950751:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a80380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+579:2388:x1870153002421632:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 211us (414us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.950756:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.950761:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c3:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.950763:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a83480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+578:2389:x1870153002421504:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 263us (621us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.950764:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36088 00010000:00010000:0.0:1783520935.950767:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025d4f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.950782:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025d4f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.950789:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36087 00010000:00010000:2.0:1783520935.950796:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025d2c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.950806:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025d2c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.950810:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.950816:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025d2c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.950824:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025d33 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.950825:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025d2c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.950837:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d322300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+576:2387:x1870153002421760:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 139us (403us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.950850:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025d33 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.950859:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36089 00010000:00010000:2.0:1783520935.950861:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.950864:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025d33 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.950872:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.950874:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025d33 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.950879:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025d2c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.950886:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025d4f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.950896:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.950911:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.950918:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025d2c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.950920:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025d4f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.950931:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025d2c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.950937:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025d4f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.950943:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025d2c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.950947:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025d4f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.950950:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025d2c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.950981:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520935.950981:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:1.0:1783520935.950986:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635785850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+427:3625:x1870152993158528:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3292us (3366us total) trans 4294975994 rc 0/0 00010000:00010000:2.0:1783520935.950987:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025d33 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.951001:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba400 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520935.951005:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44494 00010000:00010000:2.0:1783520935.951006:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025d33 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.951013:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025d33 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.951021:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025d33 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.951031:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025d33 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.951031:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025d41 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.951059:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025d41 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.951065:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.951071:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025d4f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.951071:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025d41 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.951090:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.951095:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025d4f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.951096:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025d41 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.951108:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025d4f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.951120:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025d4f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.951124:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc025d4f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.951127:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.951133:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.951136:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025d41 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.951140:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbbe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.951143:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025d41 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.951147:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025d41 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.951153:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025d41 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.951157:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025d41 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.951164:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.951166:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.951167:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.951211:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+428:3628:x1870152993158272:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3929us (4019us total) trans 4294975995 rc 0/0 00000100:00100000:1.0:1783520935.951218:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44492 00000100:00100000:2.0:1783520935.951242:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436342ba300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+427:3626:x1870152993158400:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3764us (3890us total) trans 4294975996 rc 0/0 00000100:00100000:2.0:1783520935.951247:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44493 00010000:00010000:1.0:1783520935.951255:0:35274:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cdccf04a/0xd5401cbabc0209f0 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1062 expref: 575 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.951273:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516359180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993159680:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.951294:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025d48 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.951302:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025d48 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.951310:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025d48 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.951317:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025d48 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.951324:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025d48 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.951340:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.951352:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.951379:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025d48 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520935.951394:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514f98e00/2, rc 0, transno 4294975996 00010000:00010000:3.0:1783520935.951401:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbaa00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520935.951405:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514f98e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993158400:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.951409:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025d48 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.951416:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.951420:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025d48 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.951436:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025d48 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.951447:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025d48 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.951450:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+427:3627:x1870152993159296:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2569us (2645us total) trans 4294975997 rc 0/0 00010000:00010000:0.0:1783520935.951459:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.951467:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44496 00010000:00010000:3.0:1783520935.951483:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.951487:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.951491:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520935.951509:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e11c700/2, rc 0, transno 4294975997 00010000:00010000:3.0:1783520935.951510:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.951524:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993159296:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.951534:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1783520935.951546:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435055cea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993160320:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.951560:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1783520935.951570:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435055cfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993160448:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.951581:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.951602:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:0.0:1783520935.951636:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363d321880/2, rc 0, transno 4294975994 00000100:00100000:2.0:1783520935.951646:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435055cc700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993159936:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.951653:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d321880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993158528:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.951662:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025d56 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.951663:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.951665:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a80e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993160192:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.951687:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002422016 00000100:00100000:2.0:1783520935.951691:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002422016 00000100:00100000:1.0:1783520935.951702:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.951707:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94360550ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993160576:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.951732:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:0.0:1783520935.951732:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e11ed80/2, rc 0, transno 4294975995 00010000:00010000:3.0:1783520935.951733:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025d5d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.951749:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993158272:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.951760:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993160320 00000100:00100000:1.0:1783520935.951766:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993160320 00000100:00100000:2.0:1783520935.951773:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002421888 00000100:00100000:2.0:1783520935.951778:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002421888 00010000:00010000:2.0:1783520935.951798:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000007f0d157/0xd5401cbabc020a0c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f107e expref: 576 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.951802:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993160448 00000100:00100000:0.0:1783520935.951802:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94360550ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993160704:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.951808:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993160448 00000100:00100000:2.0:1783520935.951835:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36090 00000100:00100000:1.0:1783520935.951840:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993160576 00000100:00100000:0.0:1783520935.951841:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.951844:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993160576 00000100:00100000:2.0:1783520935.951849:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94361b4a6850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+576:2389:x1870153002421888:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.951858:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.951862:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000007f0d157/0xd5401cbabc020a0c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f107e expref: 576 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.951873:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000007f0d157/0xd5401cbabc020a0c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6c4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f107e expref: 576 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.951875:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993160704 00000100:00100000:1.0:1783520935.951880:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993160704 00000100:00100000:3.0:1783520935.951882:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44500 00010000:00010000:2.0:1783520935.951891:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000007f0d157/0xd5401cbabc020a0c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6c4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f107e expref: 576 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.951896:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94360550c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+430:3627:x1870152993160320:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.951905:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.951909:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c4:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.951913:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44501 00010000:00010000:2.0:1783520935.951915:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025d56 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520935.951917:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1734 count 1 00000100:00100000:1.0:1783520935.951926:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435055cd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+431:3626:x1870152993160448:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.951928:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025d56 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.951937:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025d64 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6c6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.951938:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436342b8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+431:3626:x1870152993159424:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2854us (2959us total) trans 4294975998 rc 0/0 00002000:00080000:1.0:1783520935.951945:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1735 count 1 00010000:00010000:2.0:1783520935.951947:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.951954:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44497 00010000:00010000:3.0:1783520935.951966:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000423d7c49/0xd5401cbabc022b37 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3227 expref: 575 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.951966:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025d6b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6c7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.951977:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94361b4a6850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+575:2389:x1870153002421888:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 130us (274us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.951977:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44502 00000100:00100000:0.0:1783520935.951992:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435055cc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+430:3626:x1870152993160576:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.951995:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36090 00010000:00010000:1.0:1783520935.952001:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003541978c/0xd5401cbabc020a2f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f10a8 expref: 575 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.952012:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x6c6:0x0].0x0 since=36s v=(0 0) 00010000:00010000:1.0:1783520935.952018:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c7:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.952019:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c6:0x0].0x0 since=0s v=(1 0) 00000004:00080000:2.0:1783520935.952026:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e11c000/2, rc 0, transno 4294975998 00010000:00010000:1.0:1783520935.952027:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.952029:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025d64 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.952036:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025d6b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.952041:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993159424:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.952057:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000423d7c49/0xd5401cbabc022b37 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3227 expref: 574 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.952066:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025d56 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.952067:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1736 count 1 00010000:00010000:1.0:1783520935.952070:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003541978c/0xd5401cbabc020a2f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10a8 expref: 574 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.952072:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000423d7c49/0xd5401cbabc022b37 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3227 expref: 574 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.952081:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025d72 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6c8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.952095:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025d56 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.952104:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003541978c/0xd5401cbabc020a2f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10a8 expref: 574 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.952109:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d321f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993160832:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.952113:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007e341bbc/0xd5401cbabc020a1a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f10b6 expref: 574 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.952120:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025d56 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.952138:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c8:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.952140:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025d56 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.952142:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350384b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993160960:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.952145:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c8:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.952151:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d83c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.952155:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025d72 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.952166:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.952175:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.952177:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007e341bbc/0xd5401cbabc020a1a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10b6 expref: 574 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.952179:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025d56 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.952183:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3eb40 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.952199:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007e341bbc/0xd5401cbabc020a1a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10b6 expref: 574 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.952206:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520935.952207:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44503 00010000:00010000:2.0:1783520935.952214:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025d56 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.952218:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94360550c380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993161088:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.952222:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94360550c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+430:3627:x1870152993160704:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.952237:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025d56 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.952242:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1737 count 1 00000100:00100000:0.0:1783520935.952245:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351db63600 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.952249:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025d56 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.952257:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025d79 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6c9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.952258:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025d56 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.952274:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:1.0:1783520935.952283:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000269dc423/0xd5401cbabc020a21 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f10c4 expref: 574 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.952286:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94360550ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993161216:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.952290:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.952294:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.952299:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.952301:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c9:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.952306:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.952310:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025d79 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.952315:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.952339:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000269dc423/0xd5401cbabc020a21 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10c4 expref: 574 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.952350:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000269dc423/0xd5401cbabc020a21 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10c4 expref: 574 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.952394:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503848700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993161344:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.952447:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.952450:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+432:3625:x1870152993159552:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3358us (3420us total) trans 4294975999 rc 0/0 00000100:00100000:2.0:1783520935.952458:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44498 00000100:00100000:1.0:1783520935.952469:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993161216 00010000:00010000:3.0:1783520935.952471:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006eae17b0/0xd5401cbabc020a05 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f108c expref: 574 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.952474:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993161216 00000100:00100000:0.0:1783520935.952479:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+431:3627:x1870152993158784:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4659us (4750us total) trans 4294976000 rc 0/0 00000100:00100000:0.0:1783520935.952490:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44495 00000004:00080000:2.0:1783520935.952504:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e11e300/2, rc 0, transno 4294975999 00000100:00100000:2.0:1783520935.952520:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993159552:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.952525:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36091 00000100:00100000:1.0:1783520935.952526:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44504 00000100:00100000:3.0:1783520935.952542:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350384aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+574:2389:x1870153002422016:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.952543:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94360550d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+431:3627:x1870152993161216:12345-0@lo:6:osp-syn-1-0.0 00000004:00080000:0.0:1783520935.952544:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363d320700/2, rc 0, transno 4294976000 00000100:00100000:0.0:1783520935.952556:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993158784:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.952559:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00002000:00080000:1.0:1783520935.952566:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1738 count 1 00010000:00010000:3.0:1783520935.952567:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006eae17b0/0xd5401cbabc020a05 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f108c expref: 574 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.952576:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435055ce300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993161600:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.952582:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d80 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6ca:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.952589:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006eae17b0/0xd5401cbabc020a05 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6c5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f108c expref: 574 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.952598:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000076598846/0xd5401cbabc020a28 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f10d2 expref: 574 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.952602:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.952609:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006eae17b0/0xd5401cbabc020a05 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6c5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f108c expref: 574 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.952613:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ca:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.952621:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ca:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.952624:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993161600 00010000:00010000:3.0:1783520935.952626:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.952627:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d80 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.952628:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993161600 00000100:00100000:2.0:1783520935.952631:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:3.0:1783520935.952636:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c5:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.952643:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435055cd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993161472:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.952647:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025d5d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.952656:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44505 00010000:00010000:3.0:1783520935.952661:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025d5d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.952665:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000076598846/0xd5401cbabc020a28 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10d2 expref: 573 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.952675:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435055cce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+432:3626:x1870152993161600:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.952679:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.952693:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000076598846/0xd5401cbabc020a28 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10d2 expref: 573 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.952694:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00002000:00080000:0.0:1783520935.952697:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1740 count 1 00000100:00100000:2.0:1783520935.952710:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993161472 00000100:00100000:1.0:1783520935.952713:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503849f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993161728:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.952715:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993161472 00010000:00010000:0.0:1783520935.952716:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025d87 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6cc:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.952725:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350384aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+573:2389:x1870153002422016:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 187us (1063us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.952736:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f8fc5f73/0xd5401cbabc020a3d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f10ee expref: 573 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.952740:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36091 00000100:00100000:2.0:1783520935.952741:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44506 00000100:00100000:1.0:1783520935.952751:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e600 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.952751:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6cc:0x0].0x0 since=37s v=(0 0) 00000100:00100000:2.0:1783520935.952752:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435055cdf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+433:3626:x1870152993161472:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.952762:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6cc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.952766:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025d87 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.952769:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1739 count 1 00010000:00010000:2.0:1783520935.952784:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025d8e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6cb:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.952793:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025d5d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.952795:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f8fc5f73/0xd5401cbabc020a3d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10ee expref: 572 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.952801:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c035c542/0xd5401cbabc020a36 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f10e0 expref: 572 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.952817:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025d5d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.952826:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f8fc5f73/0xd5401cbabc020a3d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10ee expref: 572 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.952829:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6cb:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.952838:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6cb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.952843:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025d5d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.952848:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025d8e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.952860:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11e680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993161856:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.952870:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025d5d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.952891:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c035c542/0xd5401cbabc020a36 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10e0 expref: 572 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.952892:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.952896:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943515746cc0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.952910:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c035c542/0xd5401cbabc020a36 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10e0 expref: 572 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.952916:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:2.0:1783520935.952922:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993161984:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.952923:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025d5d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.952933:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.952938:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025d5d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.952944:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676840 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.952947:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025d5d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.952964:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025d5d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.952972:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025d5d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.952995:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94360550f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+434:3627:x1870152993160064:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3406us (3527us total) trans 4294976001 rc 0/0 00000100:00100000:3.0:1783520935.953004:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44499 00010000:00010000:1.0:1783520935.953007:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.953012:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.953015:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520935.953032:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350384bb80/2, rc 0, transno 4294976001 00000100:00100000:0.0:1783520935.953048:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350384bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993160064:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.953093:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d322300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993162112:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.953121:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.953145:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993162112 00000100:00100000:2.0:1783520935.953150:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993162112 00000100:00100000:2.0:1783520935.953172:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44507 00000100:00100000:2.0:1783520935.953177:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+434:3627:x1870152993162112:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.953186:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1741 count 1 00010000:00010000:2.0:1783520935.953194:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025d95 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6cd:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.953202:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004a7fed49/0xd5401cbabc020a44 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f10fc expref: 572 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.953207:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6cd:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.953209:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6cd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.953211:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025d95 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.953223:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004a7fed49/0xd5401cbabc020a44 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10fc expref: 572 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.953228:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004a7fed49/0xd5401cbabc020a44 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10fc expref: 572 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.953234:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c3b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993162240:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.953258:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435126766c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.953989:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d321f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993160832:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.954003:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025d64 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.954330:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.954401:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002422144 00000100:00100000:1.0:1783520935.954407:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002422144 00010000:00010000:1.0:1783520935.954434:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000423d7c49/0xd5401cbabc022b37 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3227 expref: 573 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.954464:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36092 00000100:00100000:1.0:1783520935.954472:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+573:2390:x1870153002422144:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.954481:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.954486:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000423d7c49/0xd5401cbabc022b37 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6c6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3227 expref: 573 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.954512:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000423d7c49/0xd5401cbabc022b37 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6c6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3227 expref: 573 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.954529:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000423d7c49/0xd5401cbabc022b37 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6c6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3227 expref: 573 pid: 35282 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.954536:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.954539:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.954542:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025d64 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.954548:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025d64 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.954562:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.954585:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025d64 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.954596:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025d64 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.954598:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+572:2390:x1870153002422144:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 126us (268us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.954606:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36092 00010000:00010000:2.0:1783520935.954618:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025d64 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.954627:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025d64 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.954647:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.954689:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.954696:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025d64 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.954706:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.954710:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025d64 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.954719:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025d64 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.954733:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025d64 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.954741:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025d64 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.954746:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94360550c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+435:3627:x1870152993160320:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2852us (3163us total) trans 4294976002 rc 0/0 00000100:00100000:2.0:1783520935.954760:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44500 00000004:00080000:1.0:1783520935.954766:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435055cea00/2, rc 0, transno 4294976002 00010000:00010000:0.0:1783520935.954773:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.954777:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.954779:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.954787:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435055cea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993160320:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.954833:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993162368:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.954864:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.954902:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993162368 00000100:00100000:2.0:1783520935.954907:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993162368 00000100:00100000:2.0:1783520935.954935:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44508 00000100:00100000:2.0:1783520935.954941:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503849180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+435:3628:x1870152993162368:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.954955:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1742 count 1 00010000:00010000:2.0:1783520935.954967:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025d9c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6ce:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.954984:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001ca951f7/0xd5401cbabc022a26 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3116 expref: 571 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.954991:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x6ce:0x0].0x0 since=36s v=(0 0) 00010000:00010000:2.0:1783520935.954994:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ce:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.954998:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025d9c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.955017:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001ca951f7/0xd5401cbabc022a26 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3116 expref: 571 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.955027:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001ca951f7/0xd5401cbabc022a26 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3116 expref: 571 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.955036:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9436342b9180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993162496:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.955075:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676540 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.955431:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350384b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993160960:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.955452:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025d6b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.955456:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94360550c380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993161088:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.955462:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.955470:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503848700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993161344:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.955489:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025d72 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.955506:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025d79 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.955508:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.955531:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002422400 00000100:00100000:2.0:1783520935.955536:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002422400 00010000:00010000:2.0:1783520935.955552:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007e341bbc/0xd5401cbabc020a1a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10b6 expref: 572 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.955575:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002422272 00000100:00100000:2.0:1783520935.955578:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002422272 00000100:00100000:0.0:1783520935.955592:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36093 00010000:00010000:2.0:1783520935.955593:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003541978c/0xd5401cbabc020a2f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10a8 expref: 573 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.955605:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436342ba300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+573:2390:x1870153002422400:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.955615:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.955621:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007e341bbc/0xd5401cbabc020a1a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10b6 expref: 573 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.955632:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007e341bbc/0xd5401cbabc020a1a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6c8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10b6 expref: 573 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.955646:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007e341bbc/0xd5401cbabc020a1a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6c8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10b6 expref: 573 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.955656:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.955664:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.955669:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025d72 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.955677:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025d72 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.955687:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36094 00000100:00100000:2.0:1783520935.955700:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436342baa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+572:2390:x1870153002422272:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.955711:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.955711:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.955718:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003541978c/0xd5401cbabc020a2f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10a8 expref: 572 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.955723:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025d72 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.955734:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003541978c/0xd5401cbabc020a2f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6c7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10a8 expref: 572 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.955738:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025d72 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.955746:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436342ba300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+572:2390:x1870153002422400:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 142us (287us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.955759:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003541978c/0xd5401cbabc020a2f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6c7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10a8 expref: 572 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.955765:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025d72 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.955767:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36093 00010000:00010000:2.0:1783520935.955781:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.955790:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.955793:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025d72 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.955797:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025d6b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.955815:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025d6b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.955820:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.955832:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.955834:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.955838:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025d72 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.955854:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.955859:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025d72 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.955861:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436342baa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+570:2390:x1870153002422272:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 162us (364us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.955868:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503849f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993161728:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.955883:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025d72 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.955889:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36094 00010000:00010000:1.0:1783520935.955893:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d80 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.955894:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.955916:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002422528 00000100:00100000:2.0:1783520935.955921:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002422528 00010000:00010000:1.0:1783520935.955922:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025d6b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.955932:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025d6b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.955938:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000269dc423/0xd5401cbabc020a21 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10c4 expref: 570 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.955946:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025d6b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.955947:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.955962:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025d6b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.955978:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002422656 00000100:00100000:2.0:1783520935.955983:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002422656 00000100:00100000:1.0:1783520935.955987:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36095 00010000:00010000:3.0:1783520935.955992:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025d72 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.956000:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000076598846/0xd5401cbabc020a28 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10d2 expref: 571 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.956002:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11f100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+571:2390:x1870153002422528:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.956013:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc025d72 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.956019:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.956026:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000269dc423/0xd5401cbabc020a21 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6c9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10c4 expref: 571 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.956041:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000269dc423/0xd5401cbabc020a21 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6c9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10c4 expref: 571 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.956041:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36096 00000100:00100000:0.0:1783520935.956054:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11f800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+571:2390:x1870153002422656:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.956062:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000269dc423/0xd5401cbabc020a21 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6c9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10c4 expref: 571 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.956072:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.956083:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000076598846/0xd5401cbabc020a28 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ca:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10d2 expref: 570 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.956084:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.956088:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.956093:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.956093:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6c9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.956097:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.956101:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025d79 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.956113:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025d79 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.956113:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000076598846/0xd5401cbabc020a28 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6ca:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10d2 expref: 570 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.956142:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000076598846/0xd5401cbabc020a28 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6ca:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10d2 expref: 570 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.956153:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.956155:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ca:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.956160:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ca:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.956164:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d80 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.956172:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025d79 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.956174:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d80 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.956184:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11f100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+569:2390:x1870153002422528:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 185us (294us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.956196:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025d79 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.956203:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36095 00010000:00010000:0.0:1783520935.956210:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.956212:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025d79 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.956221:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025d79 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.956233:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d80 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.956243:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d80 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520935.956251:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94360550ea00/2, rc 0, transno 4294976003 00010000:00010000:1.0:1783520935.956252:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d80 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.956263:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d80 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.956265:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94360550ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993160576:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.956292:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435055cc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+436:3626:x1870152993160576:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4302us (4558us total) trans 4294976003 rc 0/0 00000100:00100000:0.0:1783520935.956292:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11f800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+568:2390:x1870153002422656:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 238us (347us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.956297:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520935.956306:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44502 00000100:00100000:0.0:1783520935.956312:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36096 00010000:00010000:2.0:1783520935.956324:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.956330:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d80 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.956340:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.956344:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d80 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.956352:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d80 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.956385:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11e680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993161856:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.956391:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d80 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.956400:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025d80 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.956406:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025d87 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.956422:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94360550d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+436:3627:x1870152993161216:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3883us (4105us total) trans 4294976004 rc 0/0 00010000:00010000:2.0:1783520935.956423:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.956427:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.956430:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.956434:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44504 00000100:00100000:2.0:1783520935.956446:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9436342bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993162624:0@lo:6:osp-syn-1-0.0 00000004:00080000:1.0:1783520935.956467:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94360550ce00/2, rc 0, transno 4294976004 00000100:00100000:1.0:1783520935.956478:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94360550ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993161216:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.956490:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993161984:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.956509:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.956519:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025d8e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.956528:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.956539:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993162624 00000100:00100000:2.0:1783520935.956542:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1783520935.956544:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993162624 00000100:00100000:2.0:1783520935.956551:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503849f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993162752:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.956576:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44509 00000100:00100000:2.0:1783520935.956581:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.956589:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436342b8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+436:3625:x1870152993162624:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.956604:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1743 count 1 00000100:00100000:3.0:1783520935.956608:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993162752 00000100:00100000:3.0:1783520935.956614:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993162752 00010000:00010000:1.0:1783520935.956619:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025da3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6cf:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.956634:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000058af2b06/0xd5401cbabc020a52 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1118 expref: 567 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.956635:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.956642:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.956649:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.956650:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6cf:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.956657:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6cf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.956658:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025d79 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.956667:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb400 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520935.956672:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.956674:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025d79 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.956684:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025d79 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.956691:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44510 00010000:00010000:1.0:1783520935.956691:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025da3 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.956706:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436342b9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+437:3625:x1870152993162752:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.956708:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025d79 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.956719:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025d79 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520935.956722:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1744 count 1 00010000:00010000:1.0:1783520935.956722:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000058af2b06/0xd5401cbabc020a52 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1118 expref: 567 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.956735:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.956738:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025daa lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6d0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.956746:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.956747:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000058af2b06/0xd5401cbabc020a52 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1118 expref: 567 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.956750:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.956754:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.956755:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c5cf4b8e/0xd5401cbabc020a59 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1126 expref: 567 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.956762:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002422784 00000100:00100000:2.0:1783520935.956768:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002423040 00000100:00100000:0.0:1783520935.956769:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002422784 00000100:00100000:2.0:1783520935.956775:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002423040 00010000:00010000:3.0:1783520935.956776:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d0:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.956782:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d0:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.956786:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350384b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993162880:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.956788:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025daa lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.956792:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f8fc5f73/0xd5401cbabc020a3d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10ee expref: 569 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.956797:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004a7fed49/0xd5401cbabc020a44 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10fc expref: 569 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.956823:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c5cf4b8e/0xd5401cbabc020a59 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1126 expref: 569 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.956832:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c5cf4b8e/0xd5401cbabc020a59 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1126 expref: 569 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.956838:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002422912 00000100:00100000:3.0:1783520935.956843:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d321f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993163008:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.956844:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002422912 00000100:00100000:2.0:1783520935.956848:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002423168 00000100:00100000:1.0:1783520935.956852:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3eb40 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.956853:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002423168 00010000:00010000:0.0:1783520935.956869:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c035c542/0xd5401cbabc020a36 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10e0 expref: 571 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.956872:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8780 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.956874:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001ca951f7/0xd5401cbabc022a26 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x6ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3116 expref: 571 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.956876:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36097 00000100:00100000:1.0:1783520935.956891:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11e680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+571:2388:x1870153002422784:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.956910:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.956915:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36098 00010000:00010000:1.0:1783520935.956920:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f8fc5f73/0xd5401cbabc020a3d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6cc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10ee expref: 571 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.956925:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36099 00000100:00100000:0.0:1783520935.956929:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f0c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+571:2390:x1870153002423040:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.956940:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11c380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+571:2390:x1870153002422912:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.956945:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f8fc5f73/0xd5401cbabc020a3d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6cc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10ee expref: 571 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.956951:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.956953:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.956963:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c035c542/0xd5401cbabc020a36 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6cb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10e0 expref: 571 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.956963:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004a7fed49/0xd5401cbabc020a44 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6cd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10fc expref: 571 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.956964:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f8fc5f73/0xd5401cbabc020a3d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6cc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10ee expref: 571 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.956982:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36100 00010000:00010000:2.0:1783520935.956996:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c035c542/0xd5401cbabc020a36 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6cb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10e0 expref: 570 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.956997:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94360550c380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+570:2390:x1870153002423168:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.956998:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6cc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.957000:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004a7fed49/0xd5401cbabc020a44 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6cd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10fc expref: 570 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.957007:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6cc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.957013:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.957018:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025d87 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.957025:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001ca951f7/0xd5401cbabc022a26 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x6ce:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3116 expref: 570 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.957029:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c035c542/0xd5401cbabc020a36 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6cb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10e0 expref: 570 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.957034:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6cd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.957040:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025d87 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.957045:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6cd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.957057:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6cb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.957058:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001ca951f7/0xd5401cbabc022a26 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6ce:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3116 expref: 569 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.957058:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025d95 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6cd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.957063:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6cb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.957070:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.957073:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025d8e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.957087:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025d95 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6cd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.957088:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025d8e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.957092:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11e680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+569:2388:x1870153002422784:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 203us (569us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.957096:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ce:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520935.957105:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ce:0x0].0x0 since=0s v=(1 0) 00000100:00100000:1.0:1783520935.957114:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36097 00010000:00010000:3.0:1783520935.957117:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025d9c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ce:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.957123:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.957128:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.957130:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025d9c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6ce:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.957145:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11c380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+568:2390:x1870153002422912:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 210us (476us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.957148:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.957151:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520935.957152:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36099 00010000:00010000:0.0:1783520935.957157:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.957164:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025d6b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.957173:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.957177:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025d6b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.957178:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025d8e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.957179:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94360550c380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+567:2390:x1870153002423168:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 184us (448us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.957199:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36100 00010000:00010000:2.0:1783520935.957200:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025d8e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.957205:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025d6b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.957211:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025d8e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.957227:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025d8e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.957229:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435055cd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+438:3626:x1870152993160448:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5308us (5627us total) trans 4294976005 rc 0/0 00010000:00010000:0.0:1783520935.957242:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025d6b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.957243:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44501 00010000:00010000:2.0:1783520935.957250:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.957253:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025d6b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.957261:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.957267:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025d8e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.957270:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435055cfb80/2, rc 0, transno 4294976005 00010000:00010000:3.0:1783520935.957280:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184a00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520935.957283:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435055cfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993160448:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.957285:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.957286:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025d8e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.957290:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.957295:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.957298:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025d8e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.957313:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025d8e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.957321:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f0c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+566:2390:x1870153002423040:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 393us (681us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.957323:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435055cdf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+438:3626:x1870152993161472:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4571us (4648us total) trans 4294976006 rc 0/0 00000100:00100000:1.0:1783520935.957325:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:3.0:1783520935.957333:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025d8e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.957339:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503849880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993163136:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.957341:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44506 00000100:00100000:0.0:1783520935.957344:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36098 00010000:00010000:3.0:1783520935.957384:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.957388:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.957391:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.957411:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.957439:0:178496:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001ca951f7/0xd5401cbabc022a26 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3116 expref: 565 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.957441:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993163136 00010000:00010000:0.0:1783520935.957442:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025d87 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.957447:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993163136 00000100:00100000:2.0:1783520935.957458:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9436342b9180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993162496:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.957458:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025d87 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.957464:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435055cd500/2, rc 0, transno 4294976006 00010000:00010000:2.0:1783520935.957476:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025d9c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.957478:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025d87 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.957479:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435055cd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993161472:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.957485:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44511 00000100:00100000:3.0:1783520935.957510:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503848700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+438:3628:x1870152993163136:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.957512:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025d9c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.957514:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025d87 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520935.957529:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1745 count 1 00000100:00100000:1.0:1783520935.957536:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350384aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993163264:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.957546:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025db1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6d1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.957548:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025d9c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.957556:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.957564:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000008f8d6a7/0xd5401cbabc020a60 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1134 expref: 564 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.957570:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025d9c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.957571:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.957582:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d1:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.957587:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025d9c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.957588:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.957597:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025db1 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.957608:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.957615:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025d87 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.957623:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.957626:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000008f8d6a7/0xd5401cbabc020a60 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1134 expref: 564 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.957628:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025d87 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.957651:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025d87 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.957653:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000008f8d6a7/0xd5401cbabc020a60 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1134 expref: 564 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.957670:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025d87 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.957671:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435055cce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+439:3626:x1870152993161600:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4999us (5068us total) trans 4294976007 rc 0/0 00000100:00100000:3.0:1783520935.957673:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d321c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993163392:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.957689:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025d87 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.957689:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44505 00000100:00100000:0.0:1783520935.957711:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993163264 00000100:00100000:0.0:1783520935.957716:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993163264 00010000:00010000:2.0:1783520935.957721:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520935.957724:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435055ce300/2, rc 0, transno 4294976007 00010000:00010000:2.0:1783520935.957725:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.957731:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.957733:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8a80 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.957739:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435055ce300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993161600:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.957748:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.957754:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44512 00010000:00010000:2.0:1783520935.957760:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520935.957766:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351635b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+439:3628:x1870152993163264:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.957767:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025d9c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.957781:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.957784:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025d9c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.957786:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1746 count 1 00000100:00100000:1.0:1783520935.957792:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:2.0:1783520935.957796:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025d9c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.957804:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025db8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6d2:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.957805:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516359f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993163520:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.957811:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025d9c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.957825:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025d9c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.957828:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000cd20a934/0xd5401cbabc020a67 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1142 expref: 564 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.957842:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.957844:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.957846:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d2:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.957848:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.957852:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.957853:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.957860:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025db8 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.957864:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993163520 00000100:00100000:3.0:1783520935.957869:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993163520 00010000:00010000:0.0:1783520935.957893:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000cd20a934/0xd5401cbabc020a67 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1142 expref: 564 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.957905:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44513 00010000:00010000:0.0:1783520935.957907:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000cd20a934/0xd5401cbabc020a67 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1142 expref: 564 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.957918:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516358700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+441:3626:x1870152993163520:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.957924:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11f100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993163648:192.168.204.33@tcp:104: 00002000:00080000:3.0:1783520935.957933:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1747 count 1 00010000:00010000:3.0:1783520935.957947:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025dbf lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6d3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.957955:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503849180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+441:3628:x1870152993162368:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3010us (3086us total) trans 4294976008 rc 0/0 00000004:00080000:1.0:1783520935.957966:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363d321180/2, rc 0, transno 4294976008 00000100:00100000:0.0:1783520935.957967:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943515746cc0 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.957969:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c5865dc0/0xd5401cbabc020a6e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1150 expref: 564 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.957974:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44508 00010000:00010000:3.0:1783520935.957983:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d3:0x0].0x0 since=37s v=(0 0) 00000100:00100000:1.0:1783520935.957984:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993162368:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.957990:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.957996:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025dbf lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.958006:0:9151:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004a7fed49/0xd5401cbabc020a44 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f10fc expref: 564 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.958022:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c3b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993162240:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.958026:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c5865dc0/0xd5401cbabc020a6e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1150 expref: 563 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.958031:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1783520935.958041:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516359180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993163776:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.958044:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025d95 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.958051:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c5865dc0/0xd5401cbabc020a6e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1150 expref: 563 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.958070:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025d95 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.958077:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993163904:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.958078:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94360550c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+441:3627:x1870152993160704:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5858us (6238us total) trans 4294976009 rc 0/0 00000100:00100000:1.0:1783520935.958087:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.958090:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025d95 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.958096:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44503 00010000:00010000:0.0:1783520935.958106:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025d95 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.958115:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025d95 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.958117:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d83c0 going to sleep for 11 seconds 00000004:00080000:2.0:1783520935.958134:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94360550ed80/2, rc 0, transno 4294976009 00010000:00010000:0.0:1783520935.958135:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520935.958140:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993163776 00000100:00100000:3.0:1783520935.958145:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993163776 00000100:00100000:2.0:1783520935.958149:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94360550ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993160704:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.958149:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.958156:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025d95 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.958165:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.958170:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025d95 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.958179:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025d95 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.958193:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025d95 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.958201:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025d95 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.958205:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1783520935.958217:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9436342b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993164032:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.958225:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.958229:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.958232:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.958249:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.958256:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44514 00000100:00100000:0.0:1783520935.958260:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993164032 00000100:00100000:3.0:1783520935.958266:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d28a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+441:3625:x1870152993163776:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.958266:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993164032 00000100:00100000:0.0:1783520935.958305:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44515 00000100:00100000:0.0:1783520935.958314:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d29180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+442:3628:x1870152993164032:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.958326:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1749 count 1 00010000:00010000:0.0:1783520935.958337:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025dc6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6d5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.958349:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a12a1b2c/0xd5401cbabc020a7c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f116c expref: 563 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.958356:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d5:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.958381:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.958386:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025dc6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.958408:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a12a1b2c/0xd5401cbabc020a7c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f116c expref: 563 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.958419:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a12a1b2c/0xd5401cbabc020a7c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f116c expref: 563 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.958429:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11f800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993164160:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.958470:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943502ee5cc0 going to sleep for 11 seconds 00002000:00080000:3.0:1783520935.958570:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1748 count 1 00000100:00100000:0.0:1783520935.958575:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+443:3627:x1870152993162112:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5394us (5450us total) trans 4294976010 rc 0/0 00000100:00100000:0.0:1783520935.958584:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44507 00010000:00010000:3.0:1783520935.958586:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025dcd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6d4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.958588:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363d322300/2, rc 0, transno 4294976010 00010000:00010000:3.0:1783520935.958604:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000adb42ec7/0xd5401cbabc020a75 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f115e expref: 563 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.958612:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d322300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993162112:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.958614:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d4:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.958659:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.958665:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025dcd lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.958691:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000adb42ec7/0xd5401cbabc020a75 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f115e expref: 563 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.958701:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9436342ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993164288:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.958714:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000adb42ec7/0xd5401cbabc020a75 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f115e expref: 563 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.958726:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993164416:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.958733:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.958759:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8240 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.958781:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993164288 00000100:00100000:2.0:1783520935.958797:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993164288 00000100:00100000:2.0:1783520935.958825:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44516 00000100:00100000:2.0:1783520935.958832:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d29f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+443:3628:x1870152993164288:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.958845:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1750 count 1 00010000:00010000:2.0:1783520935.958857:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025dd4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6d6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.958869:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ff99e5ac/0xd5401cbabc022915 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3005 expref: 563 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.958876:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x6d6:0x0].0x0 since=36s v=(0 0) 00010000:00010000:2.0:1783520935.958880:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.958883:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025dd4 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.958902:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ff99e5ac/0xd5401cbabc022915 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3005 expref: 563 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.958912:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ff99e5ac/0xd5401cbabc022915 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3005 expref: 563 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.958923:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9436342baa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993164544:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.958949:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676540 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.959025:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350384b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993162880:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.959045:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025da3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.959089:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d321f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993163008:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.959106:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025daa lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.960186:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.960216:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.960235:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002423424 00000100:00100000:2.0:1783520935.960243:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002423424 00000100:00100000:3.0:1783520935.960246:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002423296 00000100:00100000:3.0:1783520935.960252:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002423296 00010000:00010000:3.0:1783520935.960275:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c5cf4b8e/0xd5401cbabc020a59 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1126 expref: 565 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.960277:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000058af2b06/0xd5401cbabc020a52 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1118 expref: 565 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.960300:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d321c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993163392:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.960302:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.960316:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36101 00010000:00010000:0.0:1783520935.960322:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025db1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.960327:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002423552 00000100:00100000:3.0:1783520935.960330:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d2a680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+565:2387:x1870153002423296:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.960334:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002423552 00010000:00010000:3.0:1783520935.960349:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.960349:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.960355:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000008f8d6a7/0xd5401cbabc020a60 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1134 expref: 566 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.960377:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c5cf4b8e/0xd5401cbabc020a59 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1126 expref: 566 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.960403:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c5cf4b8e/0xd5401cbabc020a59 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6d0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1126 expref: 566 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.960406:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11f100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993163648:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.960415:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002423680 00000100:00100000:2.0:1783520935.960421:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002423680 00010000:00010000:0.0:1783520935.960428:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025db8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.960441:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000cd20a934/0xd5401cbabc020a67 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1142 expref: 567 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.960443:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c5cf4b8e/0xd5401cbabc020a59 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6d0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1126 expref: 567 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.960460:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36102 00000100:00100000:0.0:1783520935.960474:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d29c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+566:2389:x1870153002423424:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.960475:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d0:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.960481:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36103 00010000:00010000:3.0:1783520935.960485:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.960502:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025daa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.960504:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36104 00000100:00100000:2.0:1783520935.960510:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d28000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+566:2390:x1870153002423552:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.960513:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.960519:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025daa lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.960522:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d29880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+566:2390:x1870153002423680:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.960524:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000058af2b06/0xd5401cbabc020a52 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6cf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1118 expref: 566 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.960529:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.960540:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000008f8d6a7/0xd5401cbabc020a60 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1134 expref: 566 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.960541:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.960552:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000cd20a934/0xd5401cbabc020a67 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1142 expref: 566 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.960556:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000058af2b06/0xd5401cbabc020a52 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6cf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1118 expref: 566 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.960558:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000008f8d6a7/0xd5401cbabc020a60 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6d1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1134 expref: 566 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.960559:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.960583:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000cd20a934/0xd5401cbabc020a67 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6d2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1142 expref: 566 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.960589:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000008f8d6a7/0xd5401cbabc020a60 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6d1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1134 expref: 566 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.960598:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d2a680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+566:2387:x1870153002423296:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 270us (378us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.960605:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000058af2b06/0xd5401cbabc020a52 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6cf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1118 expref: 566 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.960614:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cd20a934/0xd5401cbabc020a67 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6d2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1142 expref: 565 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.960616:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d1:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.960619:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36101 00010000:00010000:2.0:1783520935.960626:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.960627:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6cf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.960637:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6cf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.960638:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025db1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.960640:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.960645:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025da3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.960651:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d2:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.960659:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993163904:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.960661:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025db1 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.960662:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025db8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.960670:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025da3 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.960690:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025db8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.960691:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025dbf lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.960699:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.960718:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.960728:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d28000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+562:2390:x1870153002423552:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 235us (426us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.960729:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025da3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.960729:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.960743:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36103 00010000:00010000:3.0:1783520935.960745:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025da3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.960749:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d29c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+562:2389:x1870153002423424:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 277us (562us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.960760:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d29880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+561:2390:x1870153002423680:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 241us (409us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.960768:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36102 00010000:00010000:3.0:1783520935.960776:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025da3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.960781:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36104 00010000:00010000:3.0:1783520935.960789:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025da3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.960800:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025daa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.960809:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025daa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.960818:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025daa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.960825:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025daa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.960844:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.960855:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.960860:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025daa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.960867:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.960871:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025daa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.960879:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025daa lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.960892:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025daa lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.960898:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025daa lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.960921:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.960923:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.960924:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.960972:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436342b9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+444:3625:x1870152993162752:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4269us (4393us total) trans 4294976011 rc 0/0 00000100:00100000:0.0:1783520935.960980:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44510 00000004:00080000:1.0:1783520935.961002:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503849f80/2, rc 0, transno 4294976011 00010000:00010000:0.0:1783520935.961006:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025db1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.961018:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503849f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993162752:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.961018:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025db1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.961033:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025db1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.961043:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025db1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.961063:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993164672:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.961064:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.961072:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.961078:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025db1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.961087:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.961092:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025db1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.961101:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025db1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.961107:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.961119:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025db1 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.961125:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.961128:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025db1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.961137:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.961146:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025da3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.961156:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.961156:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503848700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+444:3628:x1870152993163136:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3657us (3743us total) trans 4294976012 rc 0/0 00010000:00010000:3.0:1783520935.961160:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbae00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.961160:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.961164:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.961168:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025da3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.961169:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503849880/2, rc 0, transno 4294976012 00000100:00100000:0.0:1783520935.961172:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44511 00010000:00010000:3.0:1783520935.961184:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025da3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.961191:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503849880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993163136:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.961197:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993164672 00010000:00010000:3.0:1783520935.961200:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025da3 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.961202:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993164672 00010000:00010000:3.0:1783520935.961221:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004109a0e8/0xd5401cbabc025da3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.961239:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44517 00010000:00010000:3.0:1783520935.961244:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.961248:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993164800:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.961249:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.961250:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d2bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+444:3625:x1870152993164672:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.961254:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00002000:00080000:0.0:1783520935.961270:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1751 count 1 00000100:00100000:2.0:1783520935.961278:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.961287:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025ddb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6d7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.961301:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000000e27008f/0xd5401cbabc020a91 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1188 expref: 559 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.961302:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993164800 00000100:00100000:1.0:1783520935.961308:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993164800 00010000:00010000:0.0:1783520935.961313:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d7:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.961317:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d7:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.961322:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.961326:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025ddb lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.961343:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44518 00010000:00010000:0.0:1783520935.961351:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000000e27008f/0xd5401cbabc020a91 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1188 expref: 559 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.961355:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351dc73800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+445:3626:x1870152993164800:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.961390:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.961404:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000000e27008f/0xd5401cbabc020a91 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1188 expref: 559 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.961410:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1752 count 1 00000100:00100000:0.0:1783520935.961427:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11f100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993164928:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.961429:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025de2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6d8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.961444:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436342b8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+446:3625:x1870152993162624:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4856us (4948us total) trans 4294976013 rc 0/0 00010000:00010000:1.0:1783520935.961447:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000834ffcea/0xd5401cbabc020a8a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1196 expref: 559 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.961455:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44509 00010000:00010000:1.0:1783520935.961463:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d8:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.961467:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.961472:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025de2 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.961484:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.961506:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000834ffcea/0xd5401cbabc020a8a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1196 expref: 559 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.961509:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9436342bad80/2, rc 0, transno 4294976013 00000100:00100000:0.0:1783520935.961514:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943515746cc0 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.961525:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000834ffcea/0xd5401cbabc020a8a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1196 expref: 559 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.961529:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9436342bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993162624:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.961533:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002423808 00000100:00100000:0.0:1783520935.961540:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002423808 00000100:00100000:1.0:1783520935.961546:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d2ad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993165056:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.961560:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002423936 00010000:00010000:0.0:1783520935.961564:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c5865dc0/0xd5401cbabc020a6e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1150 expref: 560 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.961570:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002423936 00000100:00100000:1.0:1783520935.961582:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e3c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.961594:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002424064 00000100:00100000:0.0:1783520935.961600:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002424064 00000100:00100000:1.0:1783520935.961609:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1783520935.961621:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36105 00000100:00100000:1.0:1783520935.961621:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506229180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993165184:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.961629:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000adb42ec7/0xd5401cbabc020a75 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f115e expref: 562 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.961636:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f0450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+562:2388:x1870153002423808:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.961650:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.961655:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.961659:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a12a1b2c/0xd5401cbabc020a7c lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f116c expref: 562 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.961660:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c5865dc0/0xd5401cbabc020a6e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1150 expref: 562 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.961665:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36106 00000100:00100000:0.0:1783520935.961682:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350622b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+562:2390:x1870153002424064:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.961691:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c5865dc0/0xd5401cbabc020a6e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6d3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1150 expref: 562 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.961692:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.961703:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000adb42ec7/0xd5401cbabc020a75 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6d4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f115e expref: 562 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.961712:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.961715:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c5865dc0/0xd5401cbabc020a6e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6d3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1150 expref: 562 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.961718:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36107 00010000:00010000:0.0:1783520935.961731:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000adb42ec7/0xd5401cbabc020a75 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f115e expref: 562 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.961735:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d3:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.961740:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506228a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+561:2390:x1870153002423936:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.961745:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.961754:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025dbf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.961756:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.961759:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.961766:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a12a1b2c/0xd5401cbabc020a7c lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6d5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f116c expref: 561 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.961769:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002424192 00010000:00010000:0.0:1783520935.961769:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d4:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.961776:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002424192 00010000:00010000:0.0:1783520935.961781:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025dcd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.961792:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025dbf lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.961798:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ff99e5ac/0xd5401cbabc022915 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x6d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3005 expref: 562 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.961801:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025dcd lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6d4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.961824:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.961835:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.961837:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36108 00000100:00100000:1.0:1783520935.961848:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9f800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+562:2387:x1870153002424192:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.961854:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f0450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+562:2388:x1870153002423808:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 220us (531us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.961861:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.961864:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350622b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+562:2390:x1870153002424064:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 185us (380us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.961869:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36105 00010000:00010000:1.0:1783520935.961870:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ff99e5ac/0xd5401cbabc022915 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x6d6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3005 expref: 562 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.961880:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36106 00010000:00010000:1.0:1783520935.961892:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ff99e5ac/0xd5401cbabc022915 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6d6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3005 expref: 561 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.961907:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d6:0x0].0x0 since=0s v=(1 0) 00000100:00100000:0.0:1783520935.961907:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993165184 00000100:00100000:0.0:1783520935.961912:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993165184 00010000:00010000:1.0:1783520935.961913:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.961920:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025dd4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.961929:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025dd4 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6d6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.961942:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44519 00010000:00010000:1.0:1783520935.961945:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.961950:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a12a1b2c/0xd5401cbabc020a7c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f116c expref: 562 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.961952:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+446:3628:x1870152993165184:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.961957:0:35282:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ff99e5ac/0xd5401cbabc022915 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3005 expref: 560 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.961970:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9f800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+560:2387:x1870153002424192:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 122us (260us total) trans 0 rc 0/0 00002000:00080000:0.0:1783520935.961972:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1753 count 1 00010000:00010000:3.0:1783520935.961981:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.961987:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d5:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.961987:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9436342baa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993164544:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.961988:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36108 00010000:00010000:0.0:1783520935.961989:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025de9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6d9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.962007:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000051b5adc0/0xd5401cbabc020a98 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f11a4 expref: 558 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.962011:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025dd4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.962017:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d9:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.962024:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.962028:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025dd4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.962031:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025de9 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.962051:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025dd4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.962062:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025dd4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.962068:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000051b5adc0/0xd5401cbabc020a98 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11a4 expref: 558 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.962076:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025dd4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.962104:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000051b5adc0/0xd5401cbabc020a98 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11a4 expref: 558 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.962109:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.962121:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11e680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993165312:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.962130:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025dc6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.962141:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025dc6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6d5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.962159:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943519219000 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.962176:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.962181:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025dd4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.962189:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.962194:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025dd4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.962202:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025dd4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.962214:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025dd4 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.962222:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025dd4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.962232:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d29f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+447:3628:x1870152993164288:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3398us (3495us total) trans 4294976014 rc 0/0 00000004:00080000:1.0:1783520935.962240:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9436342ba300/2, rc 0, transno 4294976014 00010000:00010000:0.0:1783520935.962240:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.962241:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44516 00010000:00010000:3.0:1783520935.962245:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.962245:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.962251:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.962261:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9436342ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993164288:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.962276:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11f100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993164928:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.962283:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506228a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+558:2390:x1870153002423936:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 550us (893us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.962294:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36107 00010000:00010000:0.0:1783520935.962295:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025ddb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.962312:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9436342baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993165440:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.962332:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025db8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.962340:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025db8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.962342:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.962352:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025db8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.962399:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025db8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.962416:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.962435:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025dbf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.962444:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.962447:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025dbf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.962451:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025db8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.962460:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025dbf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.962463:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.962470:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025db8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.962473:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025dbf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.962490:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025db8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.962508:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351635b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+447:3628:x1870152993163264:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4732us (4926us total) trans 4294976015 rc 0/0 00010000:00010000:3.0:1783520935.962513:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000004:00080000:1.0:1783520935.962520:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350384aa00/2, rc 0, transno 4294976015 00010000:00010000:2.0:1783520935.962524:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025db8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.962526:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44512 00000100:00100000:1.0:1783520935.962537:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350384aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993163264:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.962539:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025db8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.962544:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993165440 00000100:00100000:0.0:1783520935.962549:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993165440 00010000:00010000:2.0:1783520935.962571:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.962575:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025dbf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.962576:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44520 00000100:00100000:0.0:1783520935.962588:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e006d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+447:3628:x1870152993165440:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.962598:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.962602:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025dbf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.962603:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1754 count 1 00010000:00010000:2.0:1783520935.962615:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025dbf lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.962620:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025df0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6da:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.962627:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025dbf lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.962637:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000aded4fca/0xd5401cbabc020a9f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f11b2 expref: 557 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.962639:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025dbf lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.962656:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6da:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.962660:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6da:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.962665:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.962665:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025df0 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.962668:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.962671:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.962679:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.962694:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000aded4fca/0xd5401cbabc020a9f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11b2 expref: 557 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.962706:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000aded4fca/0xd5401cbabc020a9f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11b2 expref: 557 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.962719:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11f100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993165568:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.962724:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1783520935.962737:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94360550c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993165696:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.962764:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435192190c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.962775:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.962799:0:12734:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a12a1b2c/0xd5401cbabc020a7c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f116c expref: 557 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.962812:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11f800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993164160:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.962817:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993165696 00000100:00100000:2.0:1783520935.962824:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993165696 00010000:00010000:0.0:1783520935.962830:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025dc6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.962840:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516358700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+449:3626:x1870152993163520:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4924us (4998us total) trans 4294976016 rc 0/0 00010000:00010000:0.0:1783520935.962842:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025dc6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520935.962850:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516359f80/2, rc 0, transno 4294976016 00000100:00100000:3.0:1783520935.962856:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44513 00010000:00010000:0.0:1783520935.962859:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025dc6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.962865:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516359f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993163520:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.962870:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.962871:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025dc6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.962878:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025dc6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.962891:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.962900:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.962905:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025dc6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.962910:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.962913:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025dc6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.962916:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000adb42ec7/0xd5401cbabc020a75 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f115e expref: 556 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.962920:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025dc6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.962925:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d2ad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993165056:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.962927:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993165824:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.962949:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025dc6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.962950:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025de2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.962951:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993164416:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.962968:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025dc6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.962971:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.962980:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025dcd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.962991:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025dcd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.962993:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.962997:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.963000:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.963003:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025dcd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.963009:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002424320 00010000:00010000:3.0:1783520935.963014:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025dcd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.963014:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002424320 00010000:00010000:3.0:1783520935.963023:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025dcd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.963032:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000000e27008f/0xd5401cbabc020a91 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1188 expref: 556 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.963050:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993165824 00010000:00010000:0.0:1783520935.963054:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:2.0:1783520935.963056:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993165824 00010000:00010000:0.0:1783520935.963058:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025dcd lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.963064:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.963066:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025dcd lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.963071:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025dcd lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.963077:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025dcd lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x6d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.963083:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.963084:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44521 00010000:00010000:0.0:1783520935.963085:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.963088:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.963097:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d2a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+449:3626:x1870152993165696:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.963100:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36109 00000100:00100000:0.0:1783520935.963111:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cba8700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+556:2388:x1870153002424320:12345-192.168.204.33@tcp:103:ldlm_bl.0 00002000:00080000:2.0:1783520935.963117:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1755 count 1 00010000:00010000:0.0:1783520935.963118:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.963124:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000000e27008f/0xd5401cbabc020a91 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1188 expref: 556 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.963130:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44522 00010000:00010000:2.0:1783520935.963134:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025df7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6db:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.963138:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to regular blp list, 0 blwis in pool 00010000:00010000:0.0:1783520935.963139:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000000e27008f/0xd5401cbabc020a91 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6d7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1188 expref: 556 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.963146:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025dcd lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.963146:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d29c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+449:3626:x1870152993165824:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.963168:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000baa85205/0xd5401cbabc020aa6 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f11c0 expref: 556 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.963171:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1756 count 1 00010000:00010000:0.0:1783520935.963172:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000e27008f/0xd5401cbabc020a91 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6d7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1188 expref: 556 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.963189:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6db:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.963190:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025dfe lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6dc:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.963196:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.963197:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6db:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.963206:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.963209:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025df7 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.963213:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025ddb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.963214:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007900f17c/0xd5401cbabc020ab4 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f11ce expref: 555 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.963236:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025ddb lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.963237:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6dc:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.963244:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000baa85205/0xd5401cbabc020aa6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11c0 expref: 555 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.963247:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6dc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.963258:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025dfe lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.963261:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.963264:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000baa85205/0xd5401cbabc020aa6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11c0 expref: 555 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.963282:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d29880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993165952:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.963285:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d28a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+450:3625:x1870152993163776:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5021us (5202us total) trans 4294976017 rc 0/0 00010000:00010000:1.0:1783520935.963293:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007900f17c/0xd5401cbabc020ab4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11ce expref: 555 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.963298:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44514 00000100:00100000:0.0:1783520935.963300:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cba8700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+555:2388:x1870153002424320:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 186us (433us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.963313:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676240 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.963319:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36109 00010000:00010000:1.0:1783520935.963321:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007900f17c/0xd5401cbabc020ab4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11ce expref: 555 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.963329:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516359180/2, rc 0, transno 4294976017 00000100:00100000:1.0:1783520935.963345:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cbab100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993166080:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.963346:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516359180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993163776:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.963349:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025ddb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.963382:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025ddb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.963388:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025ddb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.963393:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025ddb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.963407:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520935.963411:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993166208:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.963414:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.963421:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025ddb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.963431:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.963435:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025ddb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.963445:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025ddb lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.963457:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.963460:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025ddb lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.963460:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e3c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.963471:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc025ddb lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.963484:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d2bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+450:3625:x1870152993164672:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2237us (2380us total) trans 4294976018 rc 0/0 00010000:00010000:2.0:1783520935.963505:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.963505:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44517 00010000:00010000:2.0:1783520935.963510:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.963513:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.963522:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993166208 00000100:00100000:0.0:1783520935.963526:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993166208 00000100:00100000:0.0:1783520935.963546:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44523 00000100:00100000:0.0:1783520935.963553:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635787050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+450:3627:x1870152993166208:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.963562:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1757 count 1 00010000:00010000:0.0:1783520935.963572:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025e05 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6dd:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.963582:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004fdff00c/0xd5401cbabc020aad lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f11dc expref: 554 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.963590:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6dd:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.963596:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6dd:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.963598:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.963605:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025e05 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.963630:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004fdff00c/0xd5401cbabc020aad lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11dc expref: 554 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.963641:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004fdff00c/0xd5401cbabc020aad lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11dc expref: 554 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.963652:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cba1c00/2, rc 0, transno 4294976018 00000100:00100000:2.0:1783520935.963652:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.963656:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11f800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993166336:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.963663:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11e680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993165312:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.963673:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993164672:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.963683:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025de9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.963691:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435192199c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.963714:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002424448 00000100:00100000:3.0:1783520935.963718:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502c9ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993166464:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.963718:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002424448 00000100:00100000:2.0:1783520935.963719:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002424576 00000100:00100000:2.0:1783520935.963727:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002424576 00010000:00010000:0.0:1783520935.963744:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000834ffcea/0xd5401cbabc020a8a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1196 expref: 556 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.963754:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.963754:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000051b5adc0/0xd5401cbabc020a98 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11a4 expref: 556 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.963783:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36110 00000100:00100000:2.0:1783520935.963792:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36111 00000100:00100000:0.0:1783520935.963796:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c9bb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+556:2389:x1870153002424448:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.963805:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520935.963807:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c9a300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+556:2390:x1870153002424576:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.963812:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000834ffcea/0xd5401cbabc020a8a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1196 expref: 556 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.963821:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.963823:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000834ffcea/0xd5401cbabc020a8a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6d8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1196 expref: 556 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.963830:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000051b5adc0/0xd5401cbabc020a98 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6d9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11a4 expref: 556 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.963849:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000834ffcea/0xd5401cbabc020a8a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6d8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1196 expref: 556 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.963852:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000051b5adc0/0xd5401cbabc020a98 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6d9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11a4 expref: 556 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.963861:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.963869:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.963873:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025de2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.963879:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000051b5adc0/0xd5401cbabc020a98 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6d9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11a4 expref: 555 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.963881:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025de2 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.963904:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.963908:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6d9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.963913:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025de9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.963916:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.963923:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025de2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.963925:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025de9 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.963943:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025de2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.963946:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c9bb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+554:2389:x1870153002424448:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 152us (348us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.963962:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.963962:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36110 00010000:00010000:1.0:1783520935.963965:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025de2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.963975:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025de2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.963988:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c9a300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+553:2390:x1870153002424576:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 184us (337us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.963993:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025de9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.963999:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36111 00000100:00100000:0.0:1783520935.964006:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11f100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993165568:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.964008:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025de9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.964022:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025df0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.964024:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993166464 00010000:00010000:1.0:1783520935.964025:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025de9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.964030:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993166464 00010000:00010000:1.0:1783520935.964040:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025de9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.964060:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.964065:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.964071:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025de9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.964080:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.964085:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025de9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.964090:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44524 00010000:00010000:2.0:1783520935.964098:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025de9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.964113:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025de9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.964121:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d29180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+453:3628:x1870152993164032:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5805us (5870us total) trans 4294976019 rc 0/0 00010000:00010000:2.0:1783520935.964127:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025de9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.964153:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.964155:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+453:3628:x1870152993165184:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2203us (2499us total) trans 4294976020 rc 0/0 00010000:00010000:2.0:1783520935.964157:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.964159:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44515 00010000:00010000:2.0:1783520935.964161:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.964163:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+453:3628:x1870152993166464:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.964174:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44519 00002000:00080000:3.0:1783520935.964178:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1758 count 1 00010000:00010000:3.0:1783520935.964190:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025e0c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6de:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.964200:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000639a00f0/0xd5401cbabc022b6f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f325f expref: 552 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.964203:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506229180/2, rc 0, transno 4294976020 00010000:00010000:3.0:1783520935.964206:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x6de:0x0].0x0 since=36s v=(0 0) 00010000:00010000:3.0:1783520935.964210:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6de:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.964211:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.964213:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025e0c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.964219:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506229180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993165184:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.964222:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.964232:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025de2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.964241:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbba00 already unused, calling callback (ffffffffc0bf9e90) 00000004:00080000:0.0:1783520935.964244:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9436342b9180/2, rc 0, transno 4294976019 00010000:00010000:3.0:1783520935.964245:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000639a00f0/0xd5401cbabc022b6f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f325f expref: 552 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.964248:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025de2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.964250:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9436342b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993164032:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.964265:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000639a00f0/0xd5401cbabc022b6f lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f325f expref: 552 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.964267:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025de2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.964289:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025de2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.964303:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025de2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.964318:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993166848:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.964322:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.964325:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.964327:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.964351:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993166592:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.964352:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.964414:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1783520935.964423:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993166720:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.964434:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351dc73800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+452:3626:x1870152993164800:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3081us (3152us total) trans 4294976021 rc 0/0 00000100:00100000:1.0:1783520935.964439:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44518 00000100:00100000:3.0:1783520935.964440:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8c00 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.964454:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993166848 00000100:00100000:2.0:1783520935.964459:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.964459:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993166848 00000100:00100000:1.0:1783520935.964478:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993166720 00000100:00100000:1.0:1783520935.964480:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993166720 00000004:00080000:2.0:1783520935.964512:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cba0e00/2, rc 0, transno 4294976021 00000100:00100000:1.0:1783520935.964516:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44525 00000100:00100000:1.0:1783520935.964524:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c98380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+453:3626:x1870152993166848:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.964528:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993164800:0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.964535:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1760 count 1 00000100:00100000:3.0:1783520935.964538:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44526 00010000:00010000:1.0:1783520935.964547:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025e13 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6e0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.964558:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000fe46ab3a/0xd5401cbabc020ac2 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1206 expref: 552 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.964564:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e0:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.964567:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.964570:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025e13 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.964586:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:1.0:1783520935.964592:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000fe46ab3a/0xd5401cbabc020ac2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1206 expref: 552 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.964596:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502c98e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993166976:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.964602:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c99180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+453:3626:x1870152993166720:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.964612:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000fe46ab3a/0xd5401cbabc020ac2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1206 expref: 552 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520935.964621:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1759 count 1 00000100:00100000:1.0:1783520935.964626:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba9180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993167104:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.964634:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.964637:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025e1a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6df:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.964651:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000cedebce6/0xd5401cbabc020ac9 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f11f8 expref: 552 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.964659:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6df:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.964663:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6df:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.964665:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.964668:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993166976 00010000:00010000:3.0:1783520935.964669:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025e1a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.964675:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993166976 00000100:00100000:2.0:1783520935.964696:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d29880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993165952:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.964702:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000cedebce6/0xd5401cbabc020ac9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11f8 expref: 552 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.964709:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cbab100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993166080:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.964710:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44527 00010000:00010000:2.0:1783520935.964711:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025df7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.964725:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c99500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+454:3628:x1870152993166976:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.964727:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000cedebce6/0xd5401cbabc020ac9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11f8 expref: 552 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.964733:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025dfe lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.964742:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1761 count 1 00000100:00100000:3.0:1783520935.964748:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c3b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993167232:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.964760:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025e21 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6e1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.964775:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a529f845/0xd5401cbabc020ad0 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1214 expref: 552 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.964777:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d80c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.964787:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e1:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.964790:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.964795:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025e21 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.964816:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a529f845/0xd5401cbabc020ad0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1214 expref: 552 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.964826:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a529f845/0xd5401cbabc020ad0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1214 expref: 552 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.964837:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11c380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993167360:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.964873:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435192190c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.965201:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.965222:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002424704 00000100:00100000:2.0:1783520935.965225:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.965269:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002424832 00000100:00100000:0.0:1783520935.965275:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002424832 00000100:00100000:3.0:1783520935.965291:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002424704 00010000:00010000:0.0:1783520935.965295:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000baa85205/0xd5401cbabc020aa6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11c0 expref: 553 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.965313:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000aded4fca/0xd5401cbabc020a9f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11b2 expref: 554 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.965331:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36112 00000100:00100000:0.0:1783520935.965343:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c98000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+554:2389:x1870153002424832:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.965349:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36113 00010000:00010000:0.0:1783520935.965354:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520935.965381:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f2450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+554:2390:x1870153002424704:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.965389:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000baa85205/0xd5401cbabc020aa6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6db:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11c0 expref: 554 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.965393:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.965401:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000aded4fca/0xd5401cbabc020a9f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6da:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11b2 expref: 554 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.965406:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000baa85205/0xd5401cbabc020aa6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6db:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11c0 expref: 554 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.965415:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.965422:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000aded4fca/0xd5401cbabc020a9f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6da:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11b2 expref: 554 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.965438:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000baa85205/0xd5401cbabc020aa6 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6db:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11c0 expref: 554 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.965443:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000aded4fca/0xd5401cbabc020a9f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6da:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11b2 expref: 554 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.965453:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11f800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993166336:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.965455:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6db:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.965461:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6da:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.965463:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6db:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.965469:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6da:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.965474:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025e05 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.965474:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025df7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.965476:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025df0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.965505:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025df0 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.965510:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025df7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.965542:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.965549:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025df0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.965551:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.965560:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025df0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.965564:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025df7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.965574:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f2450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+552:2390:x1870153002424704:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 214us (371us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.965581:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c98000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+552:2389:x1870153002424832:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 239us (351us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.965591:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025df7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.965596:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36113 00010000:00010000:1.0:1783520935.965597:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025df0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.965608:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36112 00010000:00010000:3.0:1783520935.965616:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025df7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.965618:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025df0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.965632:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002424960 00000100:00100000:0.0:1783520935.965638:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002424960 00010000:00010000:3.0:1783520935.965639:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025df7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.965650:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.965658:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004fdff00c/0xd5401cbabc020aad lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11dc expref: 551 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.965665:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520935.965672:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520935.965679:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025df0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.965685:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.965691:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025df7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.965696:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.965702:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025df0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.965705:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.965712:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025df7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.965717:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025df0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.965725:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025df7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.965734:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025df0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.965741:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e006d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+456:3628:x1870152993165440:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3155us (3399us total) trans 4294976022 rc 0/0 00010000:00010000:0.0:1783520935.965742:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025df7 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.965749:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc025df0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.965757:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44520 00010000:00010000:0.0:1783520935.965764:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025df7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.965769:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d2a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+456:3626:x1870152993165696:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2675us (2995us total) trans 4294976023 rc 0/0 00010000:00010000:2.0:1783520935.965777:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.965781:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.965782:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44521 00010000:00010000:2.0:1783520935.965785:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.965814:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.965817:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:2.0:1783520935.965820:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9436342baa00/2, rc 0, transno 4294976022 00010000:00010000:0.0:1783520935.965821:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.965834:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9436342baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993165440:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.965840:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36114 00000100:00100000:0.0:1783520935.965849:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cbab100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+551:2389:x1870153002424960:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.965859:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.965865:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004fdff00c/0xd5401cbabc020aad lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6dd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11dc expref: 551 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.965878:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004fdff00c/0xd5401cbabc020aad lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6dd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11dc expref: 551 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.965893:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004fdff00c/0xd5401cbabc020aad lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6dd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11dc expref: 551 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520935.965894:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94360550c380/2, rc 0, transno 4294976023 00000100:00100000:2.0:1783520935.965906:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94360550c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993165696:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.965908:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6dd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.965913:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6dd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.965917:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025e05 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.965926:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025e05 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.965950:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.965960:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502c9bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993167488:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.965968:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025e05 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.965978:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025e05 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.965985:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cbab100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+550:2389:x1870153002424960:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 135us (572us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.965989:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025e05 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.965996:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.965997:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502c9a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993167616:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.966001:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36114 00010000:00010000:1.0:1783520935.966010:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025e05 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.966026:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.966039:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:3.0:1783520935.966046:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.966047:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025e05 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.966058:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea000 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520935.966059:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993167488 00010000:00010000:2.0:1783520935.966065:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025e05 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.966065:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993167488 00010000:00010000:2.0:1783520935.966077:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025e05 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.966089:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025e05 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.966096:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635787050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+456:3627:x1870152993166208:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2543us (2641us total) trans 4294976024 rc 0/0 00010000:00010000:2.0:1783520935.966098:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025e05 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.966111:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44523 00010000:00010000:2.0:1783520935.966129:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.966130:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993167616 00010000:00010000:2.0:1783520935.966132:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.966135:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993167616 00010000:00010000:2.0:1783520935.966136:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:2.0:1783520935.966168:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514d2ad80/2, rc 0, transno 4294976024 00000100:00100000:1.0:1783520935.966169:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44528 00000100:00100000:2.0:1783520935.966183:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993166208:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.966184:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350384b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+456:3625:x1870152993167616:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.966202:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1763 count 1 00010000:00010000:1.0:1783520935.966226:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025e28 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6e3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.966234:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502c9aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993167744:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.966241:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000701d4691/0xd5401cbabc020ade lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1230 expref: 549 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.966251:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e3:0x0].0x0 since=37s v=(0 0) 00000100:00100000:2.0:1783520935.966252:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.966258:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.966263:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025e28 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.966273:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.966289:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000701d4691/0xd5401cbabc020ade lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1230 expref: 549 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.966302:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000701d4691/0xd5401cbabc020ade lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1230 expref: 549 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.966314:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba9c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993167872:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.966387:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e240 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.966402:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993167744 00000100:00100000:1.0:1783520935.966406:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993167744 00000100:00100000:3.0:1783520935.966419:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993166592:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.966439:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025e0c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.966441:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44529 00000100:00100000:2.0:1783520935.966452:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002425088 00000100:00100000:1.0:1783520935.966454:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a83100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+457:3626:x1870152993167744:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.966458:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002425088 00002000:00080000:1.0:1783520935.966469:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1764 count 1 00010000:00010000:2.0:1783520935.966478:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007900f17c/0xd5401cbabc020ab4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11ce expref: 550 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.966484:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025e2f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6e4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.966510:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000798c68f9/0xd5401cbabc020ae5 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f123e expref: 550 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.966517:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36115 00010000:00010000:1.0:1783520935.966523:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e4:0x0].0x0 since=37s v=(0 0) 00000100:00100000:2.0:1783520935.966529:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c9b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+550:2387:x1870153002425088:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.966530:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.966539:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025e2f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.966541:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.966549:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007900f17c/0xd5401cbabc020ab4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6dc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11ce expref: 550 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.966561:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007900f17c/0xd5401cbabc020ab4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6dc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11ce expref: 550 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.966567:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000798c68f9/0xd5401cbabc020ae5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f123e expref: 550 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.966579:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007900f17c/0xd5401cbabc020ab4 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6dc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11ce expref: 550 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.966583:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000798c68f9/0xd5401cbabc020ae5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f123e expref: 550 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.966599:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6dc:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.966602:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba8380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993168000:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.966605:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6dc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.966611:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025dfe lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.966620:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025dfe lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.966634:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e600 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.966649:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.966667:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025dfe lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.966675:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c9b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+549:2387:x1870153002425088:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 147us (421us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.966682:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025dfe lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.966688:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36115 00010000:00010000:3.0:1783520935.966700:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025dfe lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.966709:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025dfe lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.966728:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.966735:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.966742:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025dfe lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.966754:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbaa00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.966758:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025dfe lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.966767:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025dfe lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.966782:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025dfe lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.966790:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025dfe lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.966804:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.966820:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.966823:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.966825:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.966838:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.966846:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c3b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993167232:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.966863:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002425216 00010000:00010000:1.0:1783520935.966864:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025e1a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.966869:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002425216 00010000:00010000:2.0:1783520935.966887:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000639a00f0/0xd5401cbabc022b6f lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f325f expref: 549 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.966900:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba9180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993167104:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.966912:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002425472 00000100:00100000:2.0:1783520935.966917:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002425472 00010000:00010000:1.0:1783520935.966918:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025e13 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.966931:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000cedebce6/0xd5401cbabc020ac9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11f8 expref: 550 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.966936:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d29c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+458:3626:x1870152993165824:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3796us (3968us total) trans 4294976025 rc 0/0 00000100:00100000:3.0:1783520935.966950:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44522 00000100:00100000:2.0:1783520935.966967:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36116 00000100:00100000:1.0:1783520935.966975:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.966980:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c9a680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+550:2390:x1870153002425216:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.966989:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.966995:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000639a00f0/0xd5401cbabc022b6f lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6de:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f325f expref: 550 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.967006:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000639a00f0/0xd5401cbabc022b6f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6de:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f325f expref: 550 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.967025:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000639a00f0/0xd5401cbabc022b6f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6de:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f325f expref: 550 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.967036:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6de:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.967039:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6de:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.967043:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025e0c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.967051:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025e0c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.967067:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.967091:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c9a680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+549:2390:x1870153002425216:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 115us (288us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.967099:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36116 00000100:00100000:2.0:1783520935.967114:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002425600 00000100:00100000:2.0:1783520935.967117:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002425600 00010000:00010000:2.0:1783520935.967131:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a529f845/0xd5401cbabc020ad0 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1214 expref: 549 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.967146:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514d28000/2, rc 0, transno 4294976025 00000100:00100000:2.0:1783520935.967148:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36117 00000100:00100000:2.0:1783520935.967162:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c98a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+549:2390:x1870153002425472:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520935.967168:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993165824:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.967176:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.967195:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002425344 00000100:00100000:2.0:1783520935.967212:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1783520935.967220:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502c99880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993168128:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.967236:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002425344 00000100:00100000:2.0:1783520935.967249:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.967253:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000fe46ab3a/0xd5401cbabc020ac2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1206 expref: 550 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.967285:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36118 00000100:00100000:3.0:1783520935.967295:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cba9180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+550:2390:x1870153002425600:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.967299:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36119 00010000:00010000:3.0:1783520935.967308:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520935.967314:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f3c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+550:2387:x1870153002425344:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.967317:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a529f845/0xd5401cbabc020ad0 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6e1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1214 expref: 550 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.967324:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.967327:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a529f845/0xd5401cbabc020ad0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6e1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1214 expref: 550 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.967333:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000fe46ab3a/0xd5401cbabc020ac2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6e0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1206 expref: 550 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.967345:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000fe46ab3a/0xd5401cbabc020ac2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6e0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1206 expref: 550 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.967407:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000fe46ab3a/0xd5401cbabc020ac2 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6e0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1206 expref: 550 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.967419:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.967422:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.967426:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025e13 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.967430:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.967438:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025e13 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.967439:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.967449:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025e21 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.967460:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025e21 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6e1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.967462:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.967479:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.967504:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f3c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+549:2387:x1870153002425344:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 182us (318us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.967512:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36119 00000100:00100000:3.0:1783520935.967518:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cba9180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+549:2390:x1870153002425600:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 223us (545us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.967528:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36118 00010000:00010000:2.0:1783520935.967537:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.967545:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000cedebce6/0xd5401cbabc020ac9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6df:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11f8 expref: 547 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.967559:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993168128 00010000:00010000:2.0:1783520935.967559:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000cedebce6/0xd5401cbabc020ac9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6df:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11f8 expref: 547 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.967566:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993168128 00010000:00010000:2.0:1783520935.967577:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cedebce6/0xd5401cbabc020ac9 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6df:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f11f8 expref: 547 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.967589:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6df:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.967593:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6df:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.967597:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025e1a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.967602:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44530 00010000:00010000:2.0:1783520935.967610:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025e1a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.967616:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c9b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+458:3628:x1870152993168128:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520935.967639:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1765 count 1 00010000:00010000:2.0:1783520935.967640:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.967647:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba9c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993167872:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.967660:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025e36 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6e5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.967666:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025e28 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.967672:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c98a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+546:2390:x1870153002425472:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 510us (832us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.967679:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000214714f9/0xd5401cbabc020aec lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f124c expref: 546 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.967692:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36117 00010000:00010000:3.0:1783520935.967694:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e5:0x0].0x0 since=37s v=(0 0) 00000100:00100000:0.0:1783520935.967711:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba8380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993168000:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.967727:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025e2f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.967744:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025e1a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.967753:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025e1a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.967766:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025e1a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.967775:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025e1a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.967791:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.967802:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.967803:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.967806:0:8408:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a529f845/0xd5401cbabc020ad0 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1214 expref: 545 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.967811:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025e1a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.967813:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025e36 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.967825:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11c380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993167360:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.967836:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5400 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520935.967836:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.967843:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025e1a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.967850:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025e21 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.967852:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025e1a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.967858:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000214714f9/0xd5401cbabc020aec lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f124c expref: 544 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.967861:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002425728 00000100:00100000:0.0:1783520935.967868:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002425728 00010000:00010000:2.0:1783520935.967876:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025e1a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.967876:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025e21 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.967889:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000798c68f9/0xd5401cbabc020ae5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f123e expref: 545 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.967895:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc025e1a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.967900:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025e21 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.967921:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025e21 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.967923:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.967926:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025e21 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.967927:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.967930:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.967937:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36120 00000100:00100000:0.0:1783520935.967949:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+545:2387:x1870153002425728:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.967953:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.967964:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.967971:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025e21 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.967980:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.967984:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025e21 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.968002:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025e21 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.968008:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000214714f9/0xd5401cbabc020aec lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f124c expref: 545 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.968016:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025e21 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.968020:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a81500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993168256:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.968028:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc025e21 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.968030:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c99180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+460:3626:x1870152993166720:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3482us (3572us total) trans 4294976026 rc 0/0 00000100:00100000:2.0:1783520935.968044:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44526 00010000:00010000:0.0:1783520935.968057:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.968059:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c99500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+459:3628:x1870152993166976:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3337us (3425us total) trans 4294976027 rc 0/0 00010000:00010000:0.0:1783520935.968061:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.968066:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.968070:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d80c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.968077:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44527 00010000:00010000:0.0:1783520935.968078:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000004:00080000:2.0:1783520935.968081:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e11f100/2, rc 0, transno 4294976026 00010000:00010000:0.0:1783520935.968087:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000798c68f9/0xd5401cbabc020ae5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6e4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f123e expref: 545 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.968090:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025e0c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.968091:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993166720:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.968111:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025e0c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.968117:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000798c68f9/0xd5401cbabc020ae5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6e4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f123e expref: 545 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.968123:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502c98e00/2, rc 0, transno 4294976027 00000100:00100000:2.0:1783520935.968127:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:3.0:1783520935.968134:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025e0c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.968139:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502c98000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993168384:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.968142:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502c98e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993166976:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.968144:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000798c68f9/0xd5401cbabc020ae5 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6e4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f123e expref: 545 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.968147:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025e0c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.968166:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.968169:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.968173:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.968177:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025e2f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.968180:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.968186:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025e2f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.968189:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.968194:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025e0c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.968200:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.968203:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025e0c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.968204:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cbaa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993168512:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.968204:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.968217:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025e0c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.968226:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025e0c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.968232:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc025e0c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.968234:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+544:2387:x1870153002425728:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 287us (403us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.968250:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.968255:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36120 00000100:00100000:3.0:1783520935.968266:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+459:3628:x1870152993166464:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4104us (4510us total) trans 4294976028 rc 0/0 00010000:00010000:2.0:1783520935.968266:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.968270:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.968273:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.968278:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44524 00010000:00010000:0.0:1783520935.968288:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025e2f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.968296:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025e2f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.968297:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993168384 00000100:00100000:2.0:1783520935.968302:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993168384 00000100:00100000:3.0:1783520935.968303:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993168512 00000100:00100000:3.0:1783520935.968309:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993168512 00010000:00010000:0.0:1783520935.968311:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025e2f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.968314:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502c9ad80/2, rc 0, transno 4294976028 00010000:00010000:0.0:1783520935.968325:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025e2f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.968327:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44531 00000100:00100000:1.0:1783520935.968330:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502c9ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993166464:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.968339:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c9b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+460:3627:x1870152993168384:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.968345:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44532 00002000:00080000:2.0:1783520935.968353:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1766 count 1 00010000:00010000:0.0:1783520935.968355:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520935.968385:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cbab800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+460:3625:x1870152993168512:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.968385:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025e3d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6e6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.968394:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:2.0:1783520935.968404:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008910fd93/0xd5401cbabc022a5e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f314e expref: 543 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.968405:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520935.968407:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cbab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993168640:0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520935.968408:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1767 count 1 00010000:00010000:0.0:1783520935.968414:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025e2f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.968424:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x6e6:0x0].0x0 since=36s v=(0 0) 00010000:00010000:3.0:1783520935.968427:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025e44 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6e7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.968431:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.968439:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025e3d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.968447:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.968453:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.968459:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000fb635ed3/0xd5401cbabc020afa lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1268 expref: 543 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.968461:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025e2f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.968466:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e7:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.968473:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.968474:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008910fd93/0xd5401cbabc022a5e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f314e expref: 543 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.968482:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025e44 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.968482:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993168640 00010000:00010000:0.0:1783520935.968483:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025e2f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.968489:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993168640 00010000:00010000:2.0:1783520935.968506:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008910fd93/0xd5401cbabc022a5e lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f314e expref: 543 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.968525:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44533 00010000:00010000:0.0:1783520935.968526:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025e2f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.968538:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cbaaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+461:3625:x1870152993168640:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.968551:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025e2f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.968553:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000fb635ed3/0xd5401cbabc020afa lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1268 expref: 543 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.968556:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1768 count 1 00000100:00100000:2.0:1783520935.968558:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502c9a680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993168768:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.968570:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025e4b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6e8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.968581:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000fb635ed3/0xd5401cbabc020afa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1268 expref: 543 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.968583:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.968584:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000082bdd244/0xd5401cbabc020b08 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1276 expref: 543 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.968588:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.968594:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.968596:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512676540 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.968605:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e8:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.968608:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e8:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.968611:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a80380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993168896:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.968615:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025e4b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.968619:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025e13 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.968634:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025e13 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.968647:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8840 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.968648:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025e13 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.968656:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025e13 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.968664:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000082bdd244/0xd5401cbabc020b08 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1276 expref: 543 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.968669:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.968679:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000082bdd244/0xd5401cbabc020b08 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1276 expref: 543 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.968681:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.968688:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025e13 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.968697:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cbaad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993169024:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.968699:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbbe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.968706:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025e13 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.968715:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025e13 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.968725:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a83100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+462:3626:x1870152993167744:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2271us (2449us total) trans 4294976029 rc 0/0 00010000:00010000:2.0:1783520935.968732:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025e13 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.968732:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.968739:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44529 00000100:00100000:3.0:1783520935.968740:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.968746:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025e13 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.968763:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502c9aa00/2, rc 0, transno 4294976029 00000100:00100000:0.0:1783520935.968774:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002425856 00010000:00010000:2.0:1783520935.968775:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.968777:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502c9aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993167744:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.968779:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.968780:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002425856 00010000:00010000:2.0:1783520935.968786:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.968801:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000701d4691/0xd5401cbabc020ade lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1230 expref: 544 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.968834:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36121 00000100:00100000:2.0:1783520935.968835:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993169152:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.968846:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a83480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+544:2389:x1870153002425856:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.968855:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.968859:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000701d4691/0xd5401cbabc020ade lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6e3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1230 expref: 544 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.968872:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.968872:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000701d4691/0xd5401cbabc020ade lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6e3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1230 expref: 544 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.968883:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993169152 00000100:00100000:1.0:1783520935.968887:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993169152 00010000:00010000:0.0:1783520935.968888:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000701d4691/0xd5401cbabc020ade lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6e3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1230 expref: 544 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.968903:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.968907:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.968912:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025e28 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.968913:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44534 00000100:00100000:1.0:1783520935.968924:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c99c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+463:3628:x1870152993167488:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.968925:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025e28 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.968931:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44535 00002000:00080000:1.0:1783520935.968936:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1762 count 1 00010000:00010000:0.0:1783520935.968953:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.968954:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025e52 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6e2:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.968959:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c98380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+463:3626:x1870152993166848:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4436us (4609us total) trans 4294976030 rc 0/0 00000100:00100000:2.0:1783520935.968974:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44525 00010000:00010000:1.0:1783520935.968978:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000fcecd8f6/0xd5401cbabc020ad7 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1222 expref: 543 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.968983:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a83480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+543:2389:x1870153002425856:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 137us (248us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.968988:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c98a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+462:3627:x1870152993169152:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.968999:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36121 00010000:00010000:1.0:1783520935.969001:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e2:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.969006:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e2:0x0].0x0 since=0s v=(0 0) 00002000:00080000:3.0:1783520935.969009:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1769 count 1 00010000:00010000:1.0:1783520935.969011:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025e52 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.969023:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025e59 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6e9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.969026:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025e28 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.969041:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e11e680/2, rc 0, transno 4294976030 00010000:00010000:0.0:1783520935.969042:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025e28 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.969043:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000052d77b67/0xd5401cbabc020b01 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1284 expref: 542 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.969055:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993166848:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.969067:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e9:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.969070:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025e28 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.969074:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.969083:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025e59 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.969086:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025e28 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.969096:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993169280:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.969114:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.969124:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520935.969128:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.969129:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025e28 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.969139:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.969143:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025e28 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.969148:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000052d77b67/0xd5401cbabc020b01 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1284 expref: 542 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.969151:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025e28 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.969155:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993169280 00000100:00100000:2.0:1783520935.969161:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993169280 00010000:00010000:3.0:1783520935.969165:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000052d77b67/0xd5401cbabc020b01 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1284 expref: 542 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.969172:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025e28 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.969174:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000fcecd8f6/0xd5401cbabc020ad7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1222 expref: 542 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.969176:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a80e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993169408:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.969187:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44536 00000100:00100000:2.0:1783520935.969197:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cbab100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+463:3628:x1870152993169280:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.969197:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025e28 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.969203:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000fcecd8f6/0xd5401cbabc020ad7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1222 expref: 542 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.969209:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8240 going to sleep for 11 seconds 00002000:00080000:2.0:1783520935.969212:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1770 count 1 00010000:00010000:2.0:1783520935.969225:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025e60 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6ea:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.969230:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba9180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993169536:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.969237:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.969238:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001eb052e3/0xd5401cbabc020b0f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1292 expref: 542 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.969242:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.969247:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.969252:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ea:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.969255:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ea:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.969255:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.969261:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025e60 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.969278:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001eb052e3/0xd5401cbabc020b0f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1292 expref: 542 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.969284:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001eb052e3/0xd5401cbabc020b0f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1292 expref: 542 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.969292:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9f800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993169664:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.969325:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435122b6f00 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.969330:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350384b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+464:3625:x1870152993167616:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3150us (3287us total) trans 4294976031 rc 0/0 00000100:00100000:0.0:1783520935.969336:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44528 00000004:00080000:1.0:1783520935.969347:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502c9a300/2, rc 0, transno 4294976031 00000100:00100000:1.0:1783520935.969383:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502c9a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993167616:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.969424:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1783520935.969432:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993169792:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.969455:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.969508:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993169792 00000100:00100000:3.0:1783520935.969513:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993169792 00000100:00100000:3.0:1783520935.969539:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44537 00000100:00100000:3.0:1783520935.969546:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cba8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+464:3626:x1870152993169792:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520935.969559:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1771 count 1 00010000:00010000:3.0:1783520935.969571:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025e67 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6eb:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.969583:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000cfad5e30/0xd5401cbabc020b16 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f12a0 expref: 542 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.969590:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6eb:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.969593:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6eb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.969597:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025e67 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.969617:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000cfad5e30/0xd5401cbabc020b16 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12a0 expref: 542 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.969632:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000cfad5e30/0xd5401cbabc020b16 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12a0 expref: 542 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.969642:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435055cc700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993169920:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.969689:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d83c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.970559:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.970597:0:6493:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002426240 00000100:00100000:1.0:1783520935.970602:0:6493:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002426240 00000100:00100000:1.0:1783520935.970640:0:6493:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 98416 00000100:00100000:1.0:1783520935.970648:0:6493:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+38:251810:x1870153002426240:12345-192.168.204.33@tcp:101:checkfiemap.0 00010000:00010000:1.0:1783520935.970660:0:6493:0:(ldlm_lockd.c:1265:ldlm_handle_enqueue()) ### server-side enqueue handler START 00010000:00010000:1.0:1783520935.970673:0:6493:0:(ldlm_lockd.c:1343:ldlm_handle_enqueue()) ### server-side enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbba00/0xd5401cbabc025e6e lrc: 2/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT gid 0 flags: 0x40000000000000 nid: local remote: 0xc275876b859f3514 expref: -99 pid: 6493 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.970798:0:6493:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbae00/0xd5401cbabc025e75 lrc: 3/1,0 mode: --/CR res: [0x200000402:0x72a:0x0].0x0 bits 0x0/0x0 rrc: 9 type: IBT flags: 0x40000000000000 pid: 6493 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.970811:0:6493:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbae00/0xd5401cbabc025e75 lrc: 3/1,0 mode: CR/CR res: [0x200000402:0x72a:0x0].0x0 bits 0xd/0x0 rrc: 9 type: IBT flags: 0x50210000000000 pid: 6493 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.970820:0:6493:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbae00/0xd5401cbabc025e75 lrc: 3/1,0 mode: CR/CR res: [0x200000402:0x72a:0x0].0x0 bits 0xd/0x0 rrc: 9 type: IBT flags: 0x40210000000000 pid: 6493 initiator: MDT0 lru_score: 0 lru_type: 0 00000004:00200000:1.0:1783520935.970830:0:6493:0:(mdt_handler.c:909:mdt_pack_attr2body()) [0x200000402:0x72a:0x0]: returning size 262144 00010000:00010000:1.0:1783520935.970926:0:6493:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbba00/0xd5401cbabc025e6e lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT gid 0 flags: 0x44000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3514 expref: 41 pid: 6493 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.970941:0:6493:0:(ldlm_lockd.c:1513:ldlm_handle_enqueue()) ### server-side enqueue handler, sending reply (err=0, rc=0) ns: mdt-lustre-MDT0000_UUID lock: ffff943513bbae00/0xd5401cbabc025e75 lrc: 3/0,0 mode: CR/CR res: [0x200000402:0x72a:0x0].0x0 bits 0xd/0x0 rrc: 9 type: IBT gid 0 flags: 0x40200000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3514 expref: 40 pid: 6493 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.970949:0:6493:0:(ldlm_inodebits.c:74:ldlm_reprocess_inodebits_queue()) --- Reprocess resource [0x200000402:0x72a:0x0].0x0 (ffff943515473e00) 00010000:00010000:1.0:1783520935.970952:0:6493:0:(ldlm_inodebits.c:76:ldlm_reprocess_inodebits_queue()) Hint d 00010000:00010000:1.0:1783520935.970956:0:6493:0:(ldlm_lockd.c:1592:ldlm_handle_enqueue()) ### server-side enqueue handler END (lock ffff943513bbae00, rc 0) 00000100:00100000:1.0:1783520935.970989:0:6493:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+40:251810:x1870153002426240:12345-192.168.204.33@tcp:101:checkfiemap.0 Request processed in 342us (431us total) trans 8589957906 rc 0/0 00000100:00100000:1.0:1783520935.970996:0:6493:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 98416 00000100:00100000:3.0:1783520935.971760:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a81500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993168256:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.971772:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.971780:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025e36 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.971813:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502c9a680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993168768:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.971815:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002425984 00000100:00100000:0.0:1783520935.971819:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.971823:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002425984 00010000:00010000:2.0:1783520935.971839:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025e3d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.971846:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000214714f9/0xd5401cbabc020aec lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f124c expref: 543 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.971853:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002426368 00000100:00100000:3.0:1783520935.971859:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002426368 00000100:00100000:1.0:1783520935.971883:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36122 00010000:00010000:3.0:1783520935.971884:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008910fd93/0xd5401cbabc022a5e lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f314e expref: 544 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.971897:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11ce00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+544:2388:x1870153002425984:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.971907:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.971912:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000214714f9/0xd5401cbabc020aec lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6e5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f124c expref: 544 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.971920:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36123 00010000:00010000:1.0:1783520935.971930:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000214714f9/0xd5401cbabc020aec lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6e5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f124c expref: 544 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.971934:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba9180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993169536:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.971938:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.971939:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+544:2388:x1870153002426368:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.971955:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.971956:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025e52 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.971962:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008910fd93/0xd5401cbabc022a5e lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6e6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f314e expref: 544 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.971964:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000214714f9/0xd5401cbabc020aec lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6e5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f124c expref: 544 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.971976:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.971993:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e5:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.971997:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002426496 00010000:00010000:1.0:1783520935.971998:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.972001:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008910fd93/0xd5401cbabc022a5e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6e6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f314e expref: 543 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.972001:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002426624 00000100:00100000:2.0:1783520935.972003:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002426496 00010000:00010000:1.0:1783520935.972006:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025e36 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.972008:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002426624 00010000:00010000:2.0:1783520935.972027:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000082bdd244/0xd5401cbabc020b08 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1276 expref: 545 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.972029:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025e36 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.972036:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000052d77b67/0xd5401cbabc020b01 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1284 expref: 545 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.972040:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008910fd93/0xd5401cbabc022a5e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6e6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f314e expref: 545 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.972069:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.972073:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36124 00010000:00010000:3.0:1783520935.972079:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e6:0x0].0x0 since=0s v=(1 0) 00000100:00100000:0.0:1783520935.972080:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36125 00000100:00100000:2.0:1783520935.972087:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11f800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+544:2387:x1870153002426496:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.972088:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520935.972097:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025e3d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.972101:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.972104:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11c380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+544:2387:x1870153002426624:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.972105:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11ce00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+544:2388:x1870153002425984:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 207us (330us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.972111:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000082bdd244/0xd5401cbabc020b08 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6e8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1276 expref: 544 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.972122:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.972123:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025e3d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.972127:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36122 00010000:00010000:0.0:1783520935.972134:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000052d77b67/0xd5401cbabc020b01 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6e9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1284 expref: 544 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.972136:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000082bdd244/0xd5401cbabc020b08 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6e8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1276 expref: 544 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.972156:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.972161:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000052d77b67/0xd5401cbabc020b01 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6e9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1284 expref: 543 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.972167:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cbaad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993169024:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.972169:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000082bdd244/0xd5401cbabc020b08 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6e8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1276 expref: 543 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.972177:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.972181:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e9:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.972184:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+542:2388:x1870153002426368:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 252us (367us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.972185:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.972186:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025e59 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.972194:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e8:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.972205:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36123 00010000:00010000:2.0:1783520935.972206:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025e4b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.972209:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025e59 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6e9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.972228:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025e4b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.972232:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025e36 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.972237:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.972243:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025e4b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.972247:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025e36 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.972248:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.972261:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025e36 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.972263:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11c380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+541:2387:x1870153002426624:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 169us (289us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.972269:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025e4b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.972278:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11f800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+541:2387:x1870153002426496:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 193us (345us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.972285:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36125 00010000:00010000:3.0:1783520935.972286:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025e36 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.972295:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025e4b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.972307:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36124 00010000:00010000:1.0:1783520935.972317:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025e4b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.972323:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.972325:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025e4b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.972336:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.972346:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.972353:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025e36 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.972379:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.972386:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025e4b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.972390:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.972397:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025e36 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.972398:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.972405:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025e4b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.972412:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025e36 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.972416:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025e4b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.972429:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025e36 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.972434:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025e4b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.972441:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc025e36 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.972445:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.972457:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025e4b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.972473:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.972478:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.972480:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c9b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+465:3628:x1870152993168128:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4866us (5229us total) trans 4294976032 rc 0/0 00010000:00010000:2.0:1783520935.972482:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.972482:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.972487:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.972504:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.972506:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44530 00000100:00100000:2.0:1783520935.972509:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002426752 00000100:00100000:2.0:1783520935.972514:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002426752 00010000:00010000:2.0:1783520935.972533:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000fcecd8f6/0xd5401cbabc020ad7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1222 expref: 540 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.972536:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502c99880/2, rc 0, transno 4294976032 00000100:00100000:1.0:1783520935.972551:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502c99880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993168128:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.972565:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36126 00000100:00100000:2.0:1783520935.972573:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f0850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+540:2389:x1870153002426752:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520935.972616:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435055cc700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993169920:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.972629:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:3.0:1783520935.972635:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025e67 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.972636:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.972642:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cbaad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993170048:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.972660:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.972668:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000fcecd8f6/0xd5401cbabc020ad7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6e2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1222 expref: 540 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.972677:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.972686:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.972690:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000fcecd8f6/0xd5401cbabc020ad7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6e2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1222 expref: 540 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.972708:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000fcecd8f6/0xd5401cbabc020ad7 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6e2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1222 expref: 540 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.972718:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.972734:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002426880 00010000:00010000:2.0:1783520935.972734:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e2:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.972741:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002426880 00010000:00010000:2.0:1783520935.972741:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.972750:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025e52 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.972761:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025e52 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.972763:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000fb635ed3/0xd5401cbabc020afa lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1268 expref: 540 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.972781:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.972794:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002427008 00010000:00010000:2.0:1783520935.972794:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.972800:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002427008 00000100:00100000:0.0:1783520935.972810:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002427136 00000100:00100000:1.0:1783520935.972811:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cbaaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+465:3625:x1870152993168640:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4274us (4367us total) trans 4294976033 rc 0/0 00000100:00100000:0.0:1783520935.972814:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002427136 00010000:00010000:3.0:1783520935.972818:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000cfad5e30/0xd5401cbabc020b16 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12a0 expref: 541 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.972826:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44533 00000100:00100000:2.0:1783520935.972833:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f0850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+542:2389:x1870153002426752:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 256us (389us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.972834:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001eb052e3/0xd5401cbabc020b0f lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1292 expref: 542 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.972848:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36127 00000100:00100000:1.0:1783520935.972857:0:20096:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002427264 00000100:00100000:2.0:1783520935.972862:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36126 00000100:00100000:3.0:1783520935.972863:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d321c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+542:2388:x1870153002426880:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.972863:0:20096:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002427264 00010000:00010000:3.0:1783520935.972876:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.972884:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36128 00010000:00010000:3.0:1783520935.972885:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000fb635ed3/0xd5401cbabc020afa lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6e7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1268 expref: 542 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00100000:1.0:1783520935.972893:0:20096:0:(ofd_dev.c:2709:ofd_rw_hpreq_check()) @@@ lustre-OST0001 ll_ost_io00_004: refresh rw locks for [0x2c0000401:0x8f7:0x0] (65536->262143) req@ffff94350805ad80 x1870153002427264/t0(0) o4->f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff@192.168.204.33@tcp:301/0 lens 504/0 e 0 to 0 dl 1783520946 ref 1 fl New:/600/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1783520935.972899:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d321f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+542:2387:x1870153002427008:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.972916:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36129 00010000:00010000:0.0:1783520935.972916:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00002000:00100000:1.0:1783520935.972922:0:20096:0:(ofd_dev.c:2565:ofd_prolong_extent_locks()) Prolong locks for req ffff94350805ad80 with x1870153002427264 ext(65536->262143) 00010000:00010000:3.0:1783520935.972923:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000fb635ed3/0xd5401cbabc020afa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6e7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1268 expref: 542 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00010000:1.0:1783520935.972927:0:20096:0:(ofd_dev.c:2714:ofd_rw_hpreq_check()) lustre-OST0001: refreshed 0 locks timeout for req ffff94350805ad80 00010000:00010000:0.0:1783520935.972928:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000cfad5e30/0xd5401cbabc020b16 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6eb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12a0 expref: 542 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.972932:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508059f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+542:2389:x1870153002427136:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.972949:0:20096:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 34738 00010000:00010000:3.0:1783520935.972952:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.972955:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.972958:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6e7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.972959:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000cfad5e30/0xd5401cbabc020b16 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6eb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12a0 expref: 542 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.972960:0:20096:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350805ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_io00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+542:2388:x1870153002427264:12345-192.168.204.33@tcp:4:dd.0 00010000:00010000:2.0:1783520935.972965:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001eb052e3/0xd5401cbabc020b0f lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6ea:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1292 expref: 542 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.972969:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025e44 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.972996:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001eb052e3/0xd5401cbabc020b0f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ea:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1292 expref: 542 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.972998:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025e44 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6e7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.973009:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cfad5e30/0xd5401cbabc020b16 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6eb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12a0 expref: 542 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.973022:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ea:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.973027:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ea:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.973030:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.973031:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025e60 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ea:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.973031:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6eb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.973039:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6eb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.973048:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025e67 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.973049:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025e60 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6ea:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.973057:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d321c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+541:2388:x1870153002426880:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 196us (420us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.973064:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025e67 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.973076:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36127 00010000:00010000:2.0:1783520935.973083:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.973090:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.973104:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025e67 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.973107:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508059f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+540:2389:x1870153002427136:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 178us (391us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.973114:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d321f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+540:2387:x1870153002427008:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 220us (441us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.973126:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36129 00010000:00010000:3.0:1783520935.973127:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025e67 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.973133:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36128 00010000:00010000:3.0:1783520935.973143:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025e67 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.973152:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025e67 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.973154:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025e3d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.973164:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cbab480/2, rc 0, transno 4294976033 00010000:00010000:2.0:1783520935.973174:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025e3d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.973182:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.973184:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025e3d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.973193:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025e3d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.973206:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.973211:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025e67 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.973218:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.973222:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025e67 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.973230:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025e67 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.973241:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025e67 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.973248:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc025e67 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.973264:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.973267:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.973269:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.973274:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cba8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+465:3626:x1870152993169792:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3727us (3816us total) trans 4294976034 rc 0/0 00000100:00100000:3.0:1783520935.973282:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44537 00010000:00010000:2.0:1783520935.973284:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000004:00080000:0.0:1783520935.973293:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cba9c00/2, rc 0, transno 4294976034 00000100:00100000:3.0:1783520935.973299:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993170048 00000100:00100000:3.0:1783520935.973304:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993170048 00000100:00100000:3.0:1783520935.973330:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44538 00000100:00100000:3.0:1783520935.973340:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cba9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+466:3625:x1870152993170048:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520935.973355:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1772 count 1 00000100:00100000:1.0:1783520935.973355:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cbab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993168640:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.973356:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993169792:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.973387:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.973392:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025e3d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.973397:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c9b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+466:3627:x1870152993168384:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5060us (5230us total) trans 4294976035 rc 0/0 00010000:00010000:0.0:1783520935.973402:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.973408:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025e3d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.973412:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44531 00010000:00010000:3.0:1783520935.973423:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc025e83 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6ec:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.973425:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025e3d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.973430:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993170304:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.973441:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993170176:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.973450:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025e3d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.973453:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000bbc2b494/0xd5401cbabc020b1d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f12ae expref: 539 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.973462:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.973463:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025e3d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.973481:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993170304 00000100:00100000:2.0:1783520935.973487:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993170304 00010000:00010000:0.0:1783520935.973491:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.973506:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.973510:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.973522:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.973529:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ec:0x0].0x0 since=37s v=(0 0) 00000100:00100000:2.0:1783520935.973532:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993170176 00010000:00010000:3.0:1783520935.973533:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ec:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.973537:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993170176 00010000:00010000:3.0:1783520935.973540:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc025e83 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.973545:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44539 00000004:00080000:1.0:1783520935.973551:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502c98000/2, rc 0, transno 4294976035 00000100:00100000:0.0:1783520935.973558:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5bad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+467:3628:x1870152993170304:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.973561:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502c98000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993168384:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.973565:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44540 00010000:00010000:3.0:1783520935.973572:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000bbc2b494/0xd5401cbabc020b1d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12ae expref: 539 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.973577:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1774 count 1 00010000:00010000:3.0:1783520935.973593:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000bbc2b494/0xd5401cbabc020b1d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12ae expref: 539 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.973594:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025e8a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6ee:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.973605:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e99f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993170560:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.973624:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002f3de03d/0xd5401cbabc02294d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f303d expref: 539 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.973628:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435055cc700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993170432:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.973633:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5bb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+467:3627:x1870152993170176:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.973635:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.973644:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x6ee:0x0].0x0 since=36s v=(0 0) 00010000:00010000:0.0:1783520935.973648:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ee:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.973654:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025e8a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.973655:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1773 count 1 00000100:00100000:1.0:1783520935.973655:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993170560 00000100:00100000:1.0:1783520935.973659:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993170560 00010000:00010000:2.0:1783520935.973672:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025e91 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6ed:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.973685:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44541 00010000:00010000:2.0:1783520935.973688:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004a2a3d64/0xd5401cbabc020b24 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f12bc expref: 539 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.973691:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8a80 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.973694:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e98a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+468:3625:x1870152993170560:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.973702:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002f3de03d/0xd5401cbabc02294d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f303d expref: 539 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.973706:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ed:0x0].0x0 since=37s v=(0 0) 00002000:00080000:1.0:1783520935.973711:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1775 count 1 00010000:00010000:2.0:1783520935.973714:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ed:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.973724:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025e91 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.973724:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025e98 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6ef:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.973728:0:35276:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000052d77b67/0xd5401cbabc020b01 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1284 expref: 539 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.973729:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002f3de03d/0xd5401cbabc02294d lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f303d expref: 539 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.973754:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007ccdd6c6/0xd5401cbabc020b39 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f12d8 expref: 538 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.973757:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a80e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993169408:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.973763:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435080d1180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993170688:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.973768:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004a2a3d64/0xd5401cbabc020b24 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12bc expref: 538 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.973769:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ef:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.973773:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025e59 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.973776:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ef:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.973785:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025e98 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.973793:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943519219000 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.973795:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025e59 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.973796:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004a2a3d64/0xd5401cbabc020b24 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12bc expref: 538 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.973818:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025e59 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.973824:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007ccdd6c6/0xd5401cbabc020b39 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12d8 expref: 538 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.973834:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993170816:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.973843:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025e59 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.973863:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025e59 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.973866:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007ccdd6c6/0xd5401cbabc020b39 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12d8 expref: 538 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.973867:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514825600 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.973886:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9a680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993170944:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.973899:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.973908:0:12734:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001eb052e3/0xd5401cbabc020b0f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1292 expref: 538 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.973921:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9f800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993169664:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.973923:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.973941:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025e60 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.973954:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.973958:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025e59 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.973964:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.973966:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025e59 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.973973:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025e59 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.973983:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025e59 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.973990:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc025e59 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.974003:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.974005:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.974006:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.974012:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025e60 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.974020:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025e60 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.974027:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025e60 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.974034:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025e60 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.974050:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.974068:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.974073:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025e60 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.974082:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.974085:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025e60 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.974094:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025e60 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.974107:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025e60 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.974116:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025e60 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.974126:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cbab100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+469:3628:x1870152993169280:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4928us (4996us total) trans 4294976036 rc 0/0 00010000:00010000:1.0:1783520935.974133:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.974134:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44536 00010000:00010000:1.0:1783520935.974135:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.974140:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520935.974161:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9ea00/2, rc 0, transno 4294976036 00010000:00010000:2.0:1783520935.974169:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025e52 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.974170:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993169280:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.974182:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025e52 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.974195:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025e52 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.974201:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025e52 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.974218:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.974225:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:2.0:1783520935.974227:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.974232:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025e52 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.974236:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e98380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993171072:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.974245:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.974250:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025e52 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.974257:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025e52 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.974265:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.974276:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025e52 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.974284:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc025e52 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.974300:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.974303:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.974305:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.974418:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c99c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+470:3628:x1870152993167488:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5495us (8425us total) trans 4294976038 rc 0/0 00000100:00100000:2.0:1783520935.974426:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44534 00000004:00080000:1.0:1783520935.974433:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502c9bb80/2, rc 0, transno 4294976038 00000100:00100000:2.0:1783520935.974443:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993171072 00000100:00100000:1.0:1783520935.974446:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502c9bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993167488:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.974449:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993171072 00000100:00100000:2.0:1783520935.974478:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44542 00000100:00100000:2.0:1783520935.974487:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+470:3625:x1870152993171072:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.974511:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1776 count 1 00010000:00010000:2.0:1783520935.974524:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025e9f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6f0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.974524:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e98e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993171200:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.974542:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008ba9b7e3/0xd5401cbabc020b32 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f12e6 expref: 537 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.974552:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c98a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+470:3627:x1870152993169152:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5608us (5680us total) trans 4294976037 rc 0/0 00010000:00010000:2.0:1783520935.974552:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f0:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.974562:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f0:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.974565:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.974568:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025e9f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.974570:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44535 00000100:00100000:3.0:1783520935.974589:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993171200 00000100:00100000:3.0:1783520935.974595:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993171200 00000004:00080000:1.0:1783520935.974603:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cba8380/2, rc 0, transno 4294976037 00010000:00010000:2.0:1783520935.974605:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008ba9b7e3/0xd5401cbabc020b32 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12e6 expref: 537 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.974616:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993169152:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.974619:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008ba9b7e3/0xd5401cbabc020b32 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12e6 expref: 537 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.974626:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44543 00000100:00100000:3.0:1783520935.974638:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+470:3627:x1870152993171200:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.974648:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11ce00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993171328:192.168.204.33@tcp:104: 00002000:00080000:3.0:1783520935.974656:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1777 count 1 00010000:00010000:3.0:1783520935.974671:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025ea6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6f1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.974672:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435080d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993171456:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.974690:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008366e521/0xd5401cbabc020b47 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f12f4 expref: 537 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.974695:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435122b6f00 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.974698:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f1:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.974703:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.974708:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025ea6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.974719:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.974736:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008366e521/0xd5401cbabc020b47 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12f4 expref: 537 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.974749:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008366e521/0xd5401cbabc020b47 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12f4 expref: 537 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.974761:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435055cd880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993171584:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.974769:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993171456 00000100:00100000:1.0:1783520935.974775:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993171456 00000100:00100000:3.0:1783520935.974796:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8780 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.974807:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44544 00000100:00100000:1.0:1783520935.974816:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+471:3628:x1870152993171456:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.974829:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1778 count 1 00010000:00010000:3.0:1783520935.974839:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000fb635ed3/0xd5401cbabc020afa lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1268 expref: 537 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.974844:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025ead lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6f2:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.974859:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a80380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993168896:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.974864:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000056bb2959/0xd5401cbabc020b40 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1302 expref: 536 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.974872:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f2:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.974874:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025e44 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.974877:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.974885:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025ead lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.974889:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025e44 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.974902:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025e44 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.974912:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025e44 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.974915:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000056bb2959/0xd5401cbabc020b40 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1302 expref: 536 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.974929:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025e44 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.974937:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000056bb2959/0xd5401cbabc020b40 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1302 expref: 536 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.974950:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.974950:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e98700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993171712:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.974972:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.974979:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025e44 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.974988:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5c00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520935.974990:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e240 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.974993:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025e44 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.975003:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025e44 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.975018:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025e44 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.975025:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc025e44 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.975050:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.975054:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.975058:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cbab800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+472:3625:x1870152993168512:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6702us (6810us total) trans 4294976039 rc 0/0 00010000:00010000:0.0:1783520935.975058:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.975069:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44532 00000004:00080000:1.0:1783520935.975069:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cbaa680/2, rc 0, transno 4294976039 00000100:00100000:1.0:1783520935.975083:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cbaa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993168512:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.975129:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435080d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993171840:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.975154:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.975177:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993171840 00000100:00100000:0.0:1783520935.975183:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993171840 00000100:00100000:0.0:1783520935.975214:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44545 00000100:00100000:0.0:1783520935.975223:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514ff3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+472:3625:x1870152993171840:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.975236:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1779 count 1 00010000:00010000:0.0:1783520935.975247:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025eb4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6f3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.975259:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000069f91fac/0xd5401cbabc020b71 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1310 expref: 538 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.975265:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f3:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.975268:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.975271:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025eb4 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.975291:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000069f91fac/0xd5401cbabc020b71 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1310 expref: 538 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.975300:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000069f91fac/0xd5401cbabc020b71 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1310 expref: 538 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.975316:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435080d1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993171968:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.975355:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435192190c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.975451:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435080d1180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993170688:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.975468:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025e8a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.975542:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435055cc700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993170432:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.975562:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc025e83 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.975567:0:20096:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350805ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_io00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+537:2388:x1870153002427264:12345-192.168.204.33@tcp:4:dd.0 Request processed in 2606us (2785us total) trans 4294976040 rc 0/0 00000100:00100000:1.0:1783520935.975569:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993170816:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.975587:0:20096:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 34738 00002000:00100000:3.0:1783520935.975598:0:20096:0:(ofd_dev.c:2709:ofd_rw_hpreq_check()) @@@ lustre-OST0001 ll_ost_io00_004: refresh rw locks for [0x2c0000401:0x8f7:0x0] (65536->262143) req@ffff94350805ad80 x1870153002427264/t4294976040(0) o4->f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff@192.168.204.33@tcp:301/0 lens 504/448 e 0 to 0 dl 1783520946 ref 1 fl Complete:/600/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00010000:00010000:1.0:1783520935.975598:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025e91 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.975623:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00002000:00100000:3.0:1783520935.975624:0:20096:0:(ofd_dev.c:2565:ofd_prolong_extent_locks()) Prolong locks for req ffff94350805ad80 with x1870153002427264 ext(65536->262143) 00002000:00010000:3.0:1783520935.975631:0:20096:0:(ofd_dev.c:2714:ofd_rw_hpreq_check()) lustre-OST0001: refreshed 0 locks timeout for req ffff94350805ad80 00000100:00100000:0.0:1783520935.975654:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002427520 00000100:00100000:2.0:1783520935.975656:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.975661:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002427520 00010000:00010000:0.0:1783520935.975684:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000bbc2b494/0xd5401cbabc020b1d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12ae expref: 537 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.975686:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002427776 00000100:00100000:3.0:1783520935.975692:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002427776 00000100:00100000:2.0:1783520935.975694:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9a680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993170944:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.975711:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025e98 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.975716:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007ccdd6c6/0xd5401cbabc020b39 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ef:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12d8 expref: 538 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.975722:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36130 00000100:00100000:0.0:1783520935.975737:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+538:2388:x1870153002427520:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.975744:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36131 00010000:00010000:0.0:1783520935.975748:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.975757:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11c380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+538:2388:x1870153002427776:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.975758:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000bbc2b494/0xd5401cbabc020b1d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ec:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12ae expref: 538 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.975771:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.975777:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000bbc2b494/0xd5401cbabc020b1d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6ec:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12ae expref: 538 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.975781:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007ccdd6c6/0xd5401cbabc020b39 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ef:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12d8 expref: 538 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.975799:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007ccdd6c6/0xd5401cbabc020b39 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6ef:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12d8 expref: 538 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.975803:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000bbc2b494/0xd5401cbabc020b1d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6ec:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12ae expref: 538 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.975823:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ec:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.975827:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007ccdd6c6/0xd5401cbabc020b39 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6ef:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12d8 expref: 537 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.975828:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ec:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.975838:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc025e83 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.975844:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ef:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.975848:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc025e83 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.975849:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ef:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.975859:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025e98 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.975868:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025e98 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.975879:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.975898:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.975901:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc025e83 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.975912:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+536:2388:x1870153002427520:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 177us (292us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.975914:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc025e83 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.975924:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025e98 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.975927:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11c380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+536:2388:x1870153002427776:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 171us (267us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.975932:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36130 00010000:00010000:3.0:1783520935.975939:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc025e83 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.975950:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025e98 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.975950:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36131 00010000:00010000:3.0:1783520935.975961:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc025e83 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.975966:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025e98 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.975977:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025e98 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.975983:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.975992:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.975997:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc025e83 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.976010:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.976013:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc025e83 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.976020:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc025e83 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.976033:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc025e83 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.976039:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc025e83 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.976056:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.976058:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.976061:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.976076:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520935.976082:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cba9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+473:3625:x1870152993170048:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2743us (3397us total) trans 4294976041 rc 0/0 00010000:00010000:0.0:1783520935.976090:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:3.0:1783520935.976091:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44538 00010000:00010000:0.0:1783520935.976099:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025e98 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.976100:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cbaad80/2, rc 0, transno 4294976041 00010000:00010000:0.0:1783520935.976112:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.976116:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025e98 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.976117:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cbaad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993170048:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.976132:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025e98 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.976148:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025e98 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.976153:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc025e98 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.976162:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e98a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+473:3625:x1870152993170560:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2467us (2528us total) trans 4294976042 rc 0/0 00010000:00010000:0.0:1783520935.976168:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520935.976172:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e99f80/2, rc 0, transno 4294976042 00000100:00100000:2.0:1783520935.976173:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44541 00010000:00010000:0.0:1783520935.976173:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.976176:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.976183:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e99f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993170560:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.976231:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1783520935.976241:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435080d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993172096:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.976255:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435080d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993172224:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.976272:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.976292:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.976313:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993172096 00000100:00100000:2.0:1783520935.976318:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993172096 00000100:00100000:0.0:1783520935.976324:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993172224 00000100:00100000:0.0:1783520935.976330:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993172224 00000100:00100000:2.0:1783520935.976348:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44546 00000100:00100000:3.0:1783520935.976377:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44547 00000100:00100000:3.0:1783520935.976389:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435055cdf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+474:3627:x1870152993172224:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.976395:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514ff3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+474:3625:x1870152993172096:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.976409:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1780 count 1 00010000:00010000:2.0:1783520935.976422:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025ebb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6f4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520935.976427:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1781 count 1 00010000:00010000:2.0:1783520935.976435:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000090ed92ac/0xd5401cbabc020b4e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f131e expref: 534 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.976442:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f4:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.976446:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.976450:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025ebb lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.976469:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000090ed92ac/0xd5401cbabc020b4e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f131e expref: 534 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.976479:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000090ed92ac/0xd5401cbabc020b4e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f131e expref: 534 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.976488:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11f800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993172352:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.976539:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff9436298a6480 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.976689:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520935.976716:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025ec2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6f5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.976716:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002427392 00000100:00100000:0.0:1783520935.976722:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002427392 00000100:00100000:2.0:1783520935.976738:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.976745:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002f3de03d/0xd5401cbabc02294d lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f303d expref: 535 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.976765:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002427648 00000100:00100000:0.0:1783520935.976767:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002427648 00010000:00010000:0.0:1783520935.976779:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004a2a3d64/0xd5401cbabc020b24 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12bc expref: 536 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.976790:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36132 00000100:00100000:1.0:1783520935.976801:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+536:2388:x1870153002427392:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.976808:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36133 00010000:00010000:1.0:1783520935.976810:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520935.976819:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f1c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+536:2388:x1870153002427648:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.976820:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002f3de03d/0xd5401cbabc02294d lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6ee:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f303d expref: 536 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.976834:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.976839:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002f3de03d/0xd5401cbabc02294d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6ee:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f303d expref: 536 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.976842:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.976845:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004a2a3d64/0xd5401cbabc020b24 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ed:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12bc expref: 536 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.976861:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001b1c489f/0xd5401cbabc020b55 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f132c expref: 536 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.976872:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.976874:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002f3de03d/0xd5401cbabc02294d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6ee:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f303d expref: 536 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.976887:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f5:0x0].0x0 since=37s v=(0 0) 00000100:00100000:2.0:1783520935.976892:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002427904 00010000:00010000:3.0:1783520935.976897:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.976897:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004a2a3d64/0xd5401cbabc020b24 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6ed:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12bc expref: 536 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.976899:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002427904 00010000:00010000:1.0:1783520935.976902:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ee:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520935.976909:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025ec2 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.976912:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ee:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.976919:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008ba9b7e3/0xd5401cbabc020b32 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12e6 expref: 536 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.976923:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025e8a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.976929:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004a2a3d64/0xd5401cbabc020b24 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6ed:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12bc expref: 536 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.976950:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001b1c489f/0xd5401cbabc020b55 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f132c expref: 536 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.976952:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025e8a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.976957:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ed:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.976968:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ed:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.976977:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002428032 00010000:00010000:0.0:1783520935.976980:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025e91 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.976981:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001b1c489f/0xd5401cbabc020b55 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f132c expref: 535 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.976982:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002428032 00010000:00010000:1.0:1783520935.976994:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.977000:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008366e521/0xd5401cbabc020b47 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12f4 expref: 536 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.977005:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025e91 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.977010:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435055cce00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993172480:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.977027:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+536:2388:x1870153002427392:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 227us (336us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.977031:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36134 00010000:00010000:0.0:1783520935.977040:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.977042:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36132 00000100:00100000:2.0:1783520935.977046:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9a680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+536:2387:x1870153002427904:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520935.977048:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8840 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.977058:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.977064:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008ba9b7e3/0xd5401cbabc020b32 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6f0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12e6 expref: 535 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.977068:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36135 00000100:00100000:0.0:1783520935.977072:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f1c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+535:2388:x1870153002427648:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 254us (331us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.977083:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506d9f800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+535:2387:x1870153002428032:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.977092:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36133 00010000:00010000:2.0:1783520935.977093:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008ba9b7e3/0xd5401cbabc020b32 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6f0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12e6 expref: 535 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.977094:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.977104:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008366e521/0xd5401cbabc020b47 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6f1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12f4 expref: 535 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.977112:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.977118:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.977118:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008366e521/0xd5401cbabc020b47 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6f1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12f4 expref: 534 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.977128:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025e9f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.977138:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025e9f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6f0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.977139:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.977146:0:35276:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008366e521/0xd5401cbabc020b47 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6f1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12f4 expref: 534 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.977147:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.977150:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025e91 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.977159:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025ea6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f1:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.977166:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.977179:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025e91 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.977182:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025ea6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6f1:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.977191:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9a680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+534:2387:x1870153002427904:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 147us (351us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.977200:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435055cd880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993171584:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.977208:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025e91 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.977211:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36134 00010000:00010000:1.0:1783520935.977218:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.977225:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025ea6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.977225:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025e91 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.977246:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506d9f800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+532:2387:x1870153002428032:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 165us (377us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.977252:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025ea6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.977259:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36135 00000100:00100000:2.0:1783520935.977263:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.977263:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.977273:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025ea6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.977286:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.977287:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025ea6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.977287:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002428160 00000100:00100000:1.0:1783520935.977294:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002428160 00010000:00010000:2.0:1783520935.977295:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025e91 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.977304:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025ea6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.977310:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.977315:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000056bb2959/0xd5401cbabc020b40 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1302 expref: 532 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.977317:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025e91 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.977339:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025e91 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.977342:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.977350:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025ea6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.977353:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36136 00010000:00010000:2.0:1783520935.977356:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025e91 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.977387:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c9a680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+532:2388:x1870153002428160:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.977397:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025e91 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.977401:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.977401:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.977409:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025ea6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.977410:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000056bb2959/0xd5401cbabc020b40 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6f2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1302 expref: 532 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.977433:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to regular blp list, 0 blwis in pool 00010000:00010000:0.0:1783520935.977434:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025ea6 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.977436:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000056bb2959/0xd5401cbabc020b40 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6f2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1302 expref: 532 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.977448:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.977452:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.977457:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.977465:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025ea6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.977466:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.977475:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.977482:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc025ea6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.977484:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025ead lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.977517:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025ead lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6f2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.977525:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.977530:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.977534:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.977535:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.977541:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+476:3627:x1870152993171200:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2903us (2979us total) trans 4294976044 rc 0/0 00000100:00100000:3.0:1783520935.977550:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44543 00000100:00100000:1.0:1783520935.977561:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c9a680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+532:2388:x1870153002428160:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 178us (304us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.977569:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36136 00000100:00100000:0.0:1783520935.977583:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5bb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+476:3627:x1870152993170176:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4006us (4059us total) trans 4294976043 rc 0/0 00000100:00100000:0.0:1783520935.977593:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44540 00010000:00010000:1.0:1783520935.977597:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025e8a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.977602:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.977612:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002428288 00000100:00100000:0.0:1783520935.977617:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002428288 00010000:00010000:1.0:1783520935.977621:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025e8a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.977637:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025e8a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.977648:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025e8a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.977649:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.977652:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 44548 00000100:00100000:0.0:1783520935.977666:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635784050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+532:251810:x1870153002428288:12345-192.168.204.33@tcp:7:checkfiemap.0 00010000:00010000:1.0:1783520935.977679:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.977682:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.977689:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025e8a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.977699:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.977704:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025e8a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.977706:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e98e00/2, rc 0, transno 4294976044 00010000:00010000:2.0:1783520935.977717:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025e8a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.977722:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e98e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993171200:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.977732:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635784050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+532:251810:x1870153002428288:12345-192.168.204.33@tcp:7:checkfiemap.0 Request processed in 67us (130us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.977739:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025e8a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.977739:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 44548 00000004:00080000:3.0:1783520935.977749:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c5b8700/2, rc 0, transno 4294976043 00010000:00010000:2.0:1783520935.977750:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc025e8a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.977760:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993170176:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.977770:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5bad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+475:3628:x1870152993170304:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4214us (4308us total) trans 4294976045 rc 0/0 00000100:00100000:0.0:1783520935.977771:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435080d1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993171968:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.977773:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.977777:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.977782:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44539 00010000:00010000:2.0:1783520935.977783:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.977791:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025eb4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.977797:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002428416 00000100:00100000:2.0:1783520935.977802:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002428416 00000100:00100000:1.0:1783520935.977810:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435055cd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993172608:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.977822:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000069f91fac/0xd5401cbabc020b71 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1310 expref: 532 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.977837:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.977852:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36137 00000100:00100000:2.0:1783520935.977864:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350622b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+532:2390:x1870153002428416:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520935.977868:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993172608 00000004:00080000:1.0:1783520935.977872:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c5b9c00/2, rc 0, transno 4294976045 00000100:00100000:3.0:1783520935.977873:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993172608 00010000:00010000:2.0:1783520935.977873:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.977881:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000069f91fac/0xd5401cbabc020b71 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6f3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1310 expref: 532 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.977882:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993170304:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.977900:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000069f91fac/0xd5401cbabc020b71 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6f3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1310 expref: 532 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.977905:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44549 00000100:00100000:3.0:1783520935.977917:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514ff1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+475:3628:x1870152993172608:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.977923:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000069f91fac/0xd5401cbabc020b71 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6f3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1310 expref: 532 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.977928:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435055cc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993172736:0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520935.977931:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1782 count 1 00000100:00100000:0.0:1783520935.977940:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.977941:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.977945:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025ec9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6f6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.977947:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f3:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.977956:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.977958:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025eb4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.977968:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025eb4 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.977973:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1783520935.977978:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993172864:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.977989:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.977998:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.978008:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002428544 00000100:00100000:0.0:1783520935.978013:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002428544 00000100:00100000:2.0:1783520935.978020:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350622b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+532:2390:x1870153002428416:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 158us (374us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.978027:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993172736 00000100:00100000:2.0:1783520935.978028:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36137 00000100:00100000:1.0:1783520935.978033:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993172736 00010000:00010000:0.0:1783520935.978033:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000090ed92ac/0xd5401cbabc020b4e lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f131e expref: 532 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.978051:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993172864 00000100:00100000:1.0:1783520935.978054:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993172864 00000100:00100000:2.0:1783520935.978065:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44550 00000100:00100000:0.0:1783520935.978070:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36138 00000100:00100000:1.0:1783520935.978075:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44551 00000100:00100000:2.0:1783520935.978080:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+477:3625:x1870152993172736:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.978084:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435080d1f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+531:2387:x1870153002428544:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.978091:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+477:3625:x1870152993172864:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.978096:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1783 count 1 00010000:00010000:0.0:1783520935.978102:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.978105:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000da8b1520/0xd5401cbabc022ba7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3297 expref: 531 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.978111:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000090ed92ac/0xd5401cbabc020b4e lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6f4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f131e expref: 531 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.978112:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025ed0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6f7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.978112:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1784 count 1 00010000:00010000:3.0:1783520935.978121:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x6f6:0x0].0x0 since=36s v=(0 0) 00010000:00010000:1.0:1783520935.978130:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025ed7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6f8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.978131:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520935.978143:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025ec9 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.978145:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000087577f6c/0xd5401cbabc020b5c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1348 expref: 531 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.978166:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f30c9bb3/0xd5401cbabc020b63 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1356 expref: 531 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.978172:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000090ed92ac/0xd5401cbabc020b4e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6f4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f131e expref: 531 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.978173:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f7:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.978182:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.978187:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f8:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.978192:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025ed0 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.978197:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.978204:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.978207:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025ed7 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.978211:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.978220:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025ebb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.978224:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000087577f6c/0xd5401cbabc020b5c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1348 expref: 531 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.978236:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025ebb lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6f4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.978242:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f30c9bb3/0xd5401cbabc020b63 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1356 expref: 531 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.978243:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000087577f6c/0xd5401cbabc020b5c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1348 expref: 531 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.978246:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000da8b1520/0xd5401cbabc022ba7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3297 expref: 531 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.978265:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.978276:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f30c9bb3/0xd5401cbabc020b63 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1356 expref: 531 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.978278:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000da8b1520/0xd5401cbabc022ba7 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3297 expref: 531 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.978280:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506228a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993172992:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.978288:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435080d1f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+531:2387:x1870153002428544:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 206us (347us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.978305:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36138 00000100:00100000:1.0:1783520935.978311:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993173120:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.978312:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a80380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993173248:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.978331:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967a80 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.978331:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025eb4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.978340:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025eb4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.978346:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025eb4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.978352:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025eb4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.978354:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8240 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.978399:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.978407:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e3c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.978410:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.978417:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025eb4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.978418:0:8410:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000090ed92ac/0xd5401cbabc020b4e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f131e expref: 530 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.978437:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbebe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.978442:0:12734:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000056bb2959/0xd5401cbabc020b40 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1302 expref: 529 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.978443:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025eb4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.978444:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11f800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993172352:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.978464:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025ebb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.978467:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025eb4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.978472:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e98700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993171712:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.978487:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025ebb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.978490:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025eb4 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.978504:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025ead lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.978524:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025eb4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.978533:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025ead lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.978537:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025ebb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.978550:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.978553:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.978555:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.978558:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025ead lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.978562:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025ebb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.978574:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025ead lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.978577:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025ebb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.978589:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025ead lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.978602:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520935.978612:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520935.978614:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520935.978618:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.978622:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025ebb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.978625:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025ead lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.978641:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.978643:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbaa00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.978647:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025ebb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.978648:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025ead lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.978667:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025ead lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.978672:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025ebb lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.978678:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025ead lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.978687:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025ead lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.978692:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025ebb lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.978702:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025ebb lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.978707:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.978708:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514ff3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+480:3625:x1870152993172096:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2350us (2434us total) trans 4294976047 rc 0/0 00010000:00010000:0.0:1783520935.978712:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.978716:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.978719:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+480:3628:x1870152993171456:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3901us (3998us total) trans 4294976048 rc 0/0 00000100:00100000:2.0:1783520935.978725:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44546 00010000:00010000:3.0:1783520935.978732:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.978733:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44544 00010000:00010000:3.0:1783520935.978737:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.978743:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.978750:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514ff3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+478:3625:x1870152993171840:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3529us (3595us total) trans 4294976046 rc 0/0 00000100:00100000:0.0:1783520935.978757:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44545 00000004:00080000:1.0:1783520935.978775:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435080d1180/2, rc 0, transno 4294976047 00000100:00100000:1.0:1783520935.978788:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435080d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993172096:0@lo:6:osp-syn-1-0.0 00000004:00080000:1.0:1783520935.978806:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435080d2d80/2, rc 0, transno 4294976046 00000100:00100000:1.0:1783520935.978811:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435080d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993171840:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.978818:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435055cce00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993172480:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.978826:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008ba9b7e3/0xd5401cbabc020b32 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f12e6 expref: 528 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.978841:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025ec2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.978843:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11ce00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993171328:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.978865:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025e9f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.978870:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993173376:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.978877:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993173504:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.978882:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025e9f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.978890:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025e9f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.978899:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025e9f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.978907:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.978909:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025e9f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.978926:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.978928:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993173504 00000100:00100000:0.0:1783520935.978934:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993173504 00010000:00010000:2.0:1783520935.978937:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.978943:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025e9f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.978944:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435080d2a00/2, rc 0, transno 4294976048 00010000:00010000:2.0:1783520935.978954:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184000 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520935.978955:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435080d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993171456:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.978960:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025e9f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.978969:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44552 00010000:00010000:2.0:1783520935.978973:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025e9f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.978981:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+478:3628:x1870152993173504:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.978988:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025e9f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.978995:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1786 count 1 00010000:00010000:2.0:1783520935.978997:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025e9f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.979012:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025ede lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6fa:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.979015:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.979019:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.979024:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.979030:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a9aef116/0xd5401cbabc020b78 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1372 expref: 527 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.979036:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.979047:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6fa:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.979050:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6fa:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.979054:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025ede lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.979063:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.979075:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a9aef116/0xd5401cbabc020b78 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1372 expref: 527 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.979084:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993173376 00000100:00100000:2.0:1783520935.979088:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:0.0:1783520935.979089:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a9aef116/0xd5401cbabc020b78 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1372 expref: 527 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.979090:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993173376 00000100:00100000:2.0:1783520935.979099:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993173632:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.979102:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002428672 00000100:00100000:1.0:1783520935.979108:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002428672 00000100:00100000:0.0:1783520935.979109:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435080d1500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993173760:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.979127:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44553 00000100:00100000:2.0:1783520935.979128:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.979135:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001b1c489f/0xd5401cbabc020b55 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f132c expref: 528 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.979144:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a80e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+479:3627:x1870152993173376:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.979150:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435192199c0 going to sleep for 11 seconds 00002000:00080000:3.0:1783520935.979160:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1785 count 1 00000100:00100000:2.0:1783520935.979162:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993173632 00000100:00100000:2.0:1783520935.979166:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993173632 00000100:00100000:1.0:1783520935.979170:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36139 00010000:00010000:3.0:1783520935.979178:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025ee5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6f9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.979184:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d29f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+528:2387:x1870153002428672:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.979199:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e3baad83/0xd5401cbabc020b6a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1364 expref: 528 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.979199:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44554 00010000:00010000:1.0:1783520935.979199:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.979210:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001b1c489f/0xd5401cbabc020b55 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6f5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f132c expref: 528 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.979212:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d28a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+480:3626:x1870152993173632:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.979216:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f9:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.979225:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.979232:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025ee5 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.979232:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1787 count 1 00010000:00010000:1.0:1783520935.979234:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001b1c489f/0xd5401cbabc020b55 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6f5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f132c expref: 528 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.979247:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025eec lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6fb:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.979263:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e977683e/0xd5401cbabc020b7f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1380 expref: 528 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.979268:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001b1c489f/0xd5401cbabc020b55 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6f5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f132c expref: 528 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.979271:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6fb:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.979278:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6fb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.979284:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025eec lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.979289:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.979294:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.979299:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025ec2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.979302:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e977683e/0xd5401cbabc020b7f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1380 expref: 527 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.979321:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025ec2 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.979322:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e977683e/0xd5401cbabc020b7f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1380 expref: 527 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.979348:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d2bb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993173888:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.979375:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.979390:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025ec2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.979400:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025ec2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.979409:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d29f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+527:2387:x1870153002428672:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 227us (345us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.979412:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025ec2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.979421:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36139 00010000:00010000:0.0:1783520935.979428:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025ec2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.979430:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435129679c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.979454:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.979463:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.979468:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025ec2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.979475:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.979478:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025ec2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.979486:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025ec2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.979513:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025ec2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.979521:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc025ec2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.979536:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.979538:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.979540:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.979604:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+482:3625:x1870152993171072:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5118us (5340us total) trans 4294976049 rc 0/0 00000100:00100000:2.0:1783520935.979611:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44542 00000100:00100000:0.0:1783520935.979635:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435055cdf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+481:3627:x1870152993172224:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3249us (3343us total) trans 4294976050 rc 0/0 00000100:00100000:0.0:1783520935.979643:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44547 00000004:00080000:1.0:1783520935.979673:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e98380/2, rc 0, transno 4294976049 00000100:00100000:1.0:1783520935.979688:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e98380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993171072:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.979741:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993174016:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.979767:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.979791:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993174016 00000100:00100000:0.0:1783520935.979796:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993174016 00000100:00100000:0.0:1783520935.979825:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44555 00000100:00100000:0.0:1783520935.979833:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d2a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+481:3626:x1870152993174016:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.979845:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1788 count 1 00010000:00010000:0.0:1783520935.979858:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ef3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6fc:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.979870:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000df50a3e5/0xd5401cbabc020b86 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f138e expref: 526 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.979877:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6fc:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.979880:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6fc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.979884:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ef3 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.979906:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000df50a3e5/0xd5401cbabc020b86 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f138e expref: 526 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.979917:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000df50a3e5/0xd5401cbabc020b86 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f138e expref: 526 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.979928:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435080d3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993174144:192.168.204.33@tcp:104: 00000004:00080000:3.0:1783520935.979931:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435080d3100/2, rc 0, transno 4294976050 00000100:00100000:3.0:1783520935.979945:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435080d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993172224:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.979967:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435192190c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.980046:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506228a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993172992:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.980060:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993173120:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.980066:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025ed0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.980078:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025ed7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.980100:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a80380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993173248:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.980110:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a81500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993174272:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.980118:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025ec9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.980122:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435080d1500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993173760:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.980144:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.980144:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025ede lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.980247:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e3baad83/0xd5401cbabc020b6a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1364 expref: 526 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.980260:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e3baad83/0xd5401cbabc020b6a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1364 expref: 526 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.980285:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.980322:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002428928 00000100:00100000:2.0:1783520935.980328:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002428928 00010000:00010000:2.0:1783520935.980348:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000087577f6c/0xd5401cbabc020b5c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1348 expref: 527 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.980349:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.980379:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a83480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993174400:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.980431:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8840 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.980450:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993174272 00000100:00100000:1.0:1783520935.980455:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993174272 00000100:00100000:2.0:1783520935.980473:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002429184 00000100:00100000:2.0:1783520935.980478:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002429184 00000100:00100000:1.0:1783520935.980489:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44556 00010000:00010000:2.0:1783520935.980511:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a9aef116/0xd5401cbabc020b78 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1372 expref: 528 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.980513:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a80380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+482:3627:x1870152993174272:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.980530:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36140 00000100:00100000:0.0:1783520935.980541:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f1450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+528:2387:x1870153002428928:12345-192.168.204.33@tcp:103:ldlm_bl.0 00002000:00080000:1.0:1783520935.980549:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1789 count 1 00010000:00010000:0.0:1783520935.980550:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520935.980555:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36141 00010000:00010000:0.0:1783520935.980559:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000087577f6c/0xd5401cbabc020b5c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6f7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1348 expref: 528 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.980569:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435080d1500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+528:2387:x1870153002429184:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.980569:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025efa lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6fd:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.980572:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000087577f6c/0xd5401cbabc020b5c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6f7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1348 expref: 528 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.980586:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.980597:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a9aef116/0xd5401cbabc020b78 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6fa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1372 expref: 528 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.980603:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000087577f6c/0xd5401cbabc020b5c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6f7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1348 expref: 528 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.980604:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009f4ae15d/0xd5401cbabc020b8d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f139c expref: 528 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.980619:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a9aef116/0xd5401cbabc020b78 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6fa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1372 expref: 528 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.980630:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6fd:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.980631:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.980639:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6fd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.980640:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.980646:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a9aef116/0xd5401cbabc020b78 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6fa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1372 expref: 527 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.980648:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025efa lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.980650:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025ed0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.980657:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d2bb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993173888:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.980673:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6fa:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.980680:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025eec lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.980681:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6fa:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.980692:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009f4ae15d/0xd5401cbabc020b8d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f139c expref: 526 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.980695:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025ede lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.980695:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025ed0 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.980724:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025ede lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.980724:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009f4ae15d/0xd5401cbabc020b8d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f139c expref: 526 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.980737:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.980748:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025ed0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.980762:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.980763:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025ed0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.980765:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff0e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993174528:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.980772:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f1450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+526:2387:x1870153002428928:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 229us (482us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.980787:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435080d1500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+526:2387:x1870153002429184:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 222us (440us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.980789:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025ed0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.980790:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36140 00010000:00010000:3.0:1783520935.980805:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025ed0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.980805:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36141 00000100:00100000:1.0:1783520935.980805:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e240 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.980832:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.980842:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.980847:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025ed0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.980854:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.980859:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025ed0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.980862:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.980870:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025ed0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.980883:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025ed0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.980891:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025ed0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.980909:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.980910:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.980914:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.980916:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.980936:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002428800 00000100:00100000:0.0:1783520935.980941:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002428800 00000100:00100000:3.0:1783520935.980942:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+483:3625:x1870152993172736:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2864us (2984us total) trans 4294976051 rc 0/0 00000100:00100000:3.0:1783520935.980951:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44550 00010000:00010000:0.0:1783520935.980963:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000da8b1520/0xd5401cbabc022ba7 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3297 expref: 525 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.980972:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.980998:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002429312 00000100:00100000:0.0:1783520935.980998:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002429056 00000100:00100000:0.0:1783520935.981003:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002429056 00000100:00100000:3.0:1783520935.981005:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002429312 00010000:00010000:0.0:1783520935.981021:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f30c9bb3/0xd5401cbabc020b63 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1356 expref: 526 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.981044:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36142 00000100:00100000:0.0:1783520935.981052:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36143 00000100:00100000:2.0:1783520935.981058:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d29c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+526:2387:x1870153002428800:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.981068:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d29880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+526:2387:x1870153002429056:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.981069:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.981079:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000da8b1520/0xd5401cbabc022ba7 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6f6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3297 expref: 527 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.981082:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.981092:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f30c9bb3/0xd5401cbabc020b63 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6f8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1356 expref: 527 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.981097:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000da8b1520/0xd5401cbabc022ba7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6f6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3297 expref: 527 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.981105:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f30c9bb3/0xd5401cbabc020b63 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6f8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1356 expref: 527 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.981122:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000da8b1520/0xd5401cbabc022ba7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6f6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3297 expref: 527 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.981124:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f30c9bb3/0xd5401cbabc020b63 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6f8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1356 expref: 527 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.981147:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.981150:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.981152:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.981156:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.981159:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025ec9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.981163:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025ed7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.981172:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e977683e/0xd5401cbabc020b7f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1380 expref: 527 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.981174:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025ec9 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.981184:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025ed7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.981209:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.981211:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36144 00010000:00010000:0.0:1783520935.981218:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.981220:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d29f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+525:2387:x1870153002429312:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.981229:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520935.981229:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d29c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+525:2387:x1870153002428800:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 176us (371us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.981236:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e977683e/0xd5401cbabc020b7f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6fb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1380 expref: 525 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.981248:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36142 00000100:00100000:0.0:1783520935.981249:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d29880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+525:2387:x1870153002429056:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 182us (337us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.981254:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e977683e/0xd5401cbabc020b7f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6fb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1380 expref: 525 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.981265:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36143 00010000:00010000:2.0:1783520935.981269:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025ec9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.981271:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e977683e/0xd5401cbabc020b7f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6fb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1380 expref: 524 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.981286:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025ed7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.981288:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025ec9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.981293:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6fb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.981302:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6fb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.981310:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025ed7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.981312:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025eec lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.981313:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025ec9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.981326:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025ed7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.981338:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025ec9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.981340:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025eec lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.981346:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025ed7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.981401:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520935.981409:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.981411:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520935.981423:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520935.981424:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.981430:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025ec9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.981432:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025ed7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.981448:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.981451:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.981453:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025ec9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.981459:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025ed7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.981461:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025ec9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.981477:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+483:3625:x1870152993172864:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3390us (3476us total) trans 4294976052 rc 0/0 00010000:00010000:2.0:1783520935.981484:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025ec9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.981506:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44551 00010000:00010000:2.0:1783520935.981513:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc025ec9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.981534:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.981537:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.981537:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025ede lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.981540:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.981556:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025ede lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.981565:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025ede lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.981571:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025ed7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.981577:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025ede lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.981594:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025ed7 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.981599:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514ff3100/2, rc 0, transno 4294976052 00010000:00010000:0.0:1783520935.981600:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.981601:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc025ed7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.981613:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.981620:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025ede lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.981621:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993172864:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.981627:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.981631:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.981633:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514ff1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+483:3628:x1870152993172608:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3715us (3792us total) trans 4294976053 rc 0/0 00010000:00010000:3.0:1783520935.981635:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.981639:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb600 already unused, calling callback (ffffffffc0bf9e90) 00000004:00080000:1.0:1783520935.981642:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435055cc700/2, rc 0, transno 4294976051 00010000:00010000:0.0:1783520935.981645:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025ede lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.981648:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435055cc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993172736:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.981649:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44549 00010000:00010000:0.0:1783520935.981662:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025ede lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.981666:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d29f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+522:2387:x1870153002429312:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 446us (697us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.981681:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36144 00010000:00010000:0.0:1783520935.981682:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025ede lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520935.981690:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435055cd880/2, rc 0, transno 4294976053 00010000:00010000:0.0:1783520935.981693:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025ede lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.981700:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435055cd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993172608:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.981709:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025eec lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.981715:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.981718:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993174656:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.981720:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.981725:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.981726:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025eec lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.981737:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025eec lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.981745:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025eec lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.981757:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.981769:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993174784:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.981772:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.981778:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.981785:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025eec lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.981793:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993174912:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.981795:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.981801:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025eec lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.981810:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025eec lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.981822:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.981829:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025eec lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.981837:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993174656 00010000:00010000:0.0:1783520935.981839:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025eec lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.981845:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993174656 00000100:00100000:2.0:1783520935.981862:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.981865:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.981868:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.981871:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.981887:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993174784 00000100:00100000:1.0:1783520935.981893:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993174784 00000100:00100000:2.0:1783520935.981900:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993174912 00000100:00100000:2.0:1783520935.981904:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993174912 00000100:00100000:1.0:1783520935.981931:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44557 00000100:00100000:2.0:1783520935.981938:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44558 00000100:00100000:1.0:1783520935.981944:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514ff1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+487:3625:x1870152993174784:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.981949:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350622b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+487:3626:x1870152993174912:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.981953:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+487:3628:x1870152993173504:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2972us (3042us total) trans 4294976054 rc 0/0 00000100:00100000:0.0:1783520935.981962:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44552 00002000:00080000:1.0:1783520935.981965:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1791 count 1 00000100:00100000:3.0:1783520935.981971:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d28a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+487:3626:x1870152993173632:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2759us (2842us total) trans 4294976055 rc 0/0 00002000:00080000:2.0:1783520935.981974:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1792 count 1 00010000:00010000:1.0:1783520935.981984:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025f01 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6ff:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.981986:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44554 00000004:00080000:0.0:1783520935.981996:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514ff2300/2, rc 0, transno 4294976054 00010000:00010000:2.0:1783520935.981998:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025f08 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x700:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.982003:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005b5ab580/0xd5401cbabc020b94 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x6ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f13b8 expref: 521 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.982016:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993173504:0@lo:6:osp-syn-1-0.0 00000004:00080000:3.0:1783520935.982026:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514ff3b80/2, rc 0, transno 4294976055 00010000:00010000:1.0:1783520935.982031:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ff:0x0].0x0 since=37s v=(0 0) 00000100:00100000:3.0:1783520935.982036:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993173632:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.982038:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000010fcf996/0xd5401cbabc020b9b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x700:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f13c6 expref: 521 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.982039:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ff:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.982049:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025f01 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.982050:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x700:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.982055:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x700:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.982062:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025f08 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x700:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.982101:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000010fcf996/0xd5401cbabc020b9b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x700:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13c6 expref: 521 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.982114:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000010fcf996/0xd5401cbabc020b9b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x700:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13c6 expref: 521 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.982126:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506228a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993175040:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.982147:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993175296:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.982170:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005b5ab580/0xd5401cbabc020b94 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13b8 expref: 521 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.982173:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993175168:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.982173:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967a80 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.982185:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.982205:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005b5ab580/0xd5401cbabc020b94 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13b8 expref: 521 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.982219:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9a680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993175424:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.982221:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993175296 00000100:00100000:3.0:1783520935.982225:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.982227:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993175296 00000100:00100000:0.0:1783520935.982230:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44559 00000100:00100000:0.0:1783520935.982242:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d2bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+486:3628:x1870152993174656:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.982257:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1790 count 1 00000100:00100000:2.0:1783520935.982258:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993175168 00000100:00100000:2.0:1783520935.982262:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993175168 00010000:00010000:0.0:1783520935.982272:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f0f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6fe:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.982275:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3eb40 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.982277:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44560 00000100:00100000:3.0:1783520935.982292:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435080d1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+487:3628:x1870152993175296:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.982294:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000801985f0/0xd5401cbabc022a96 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6fe:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3186 expref: 521 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.982302:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44561 00010000:00010000:0.0:1783520935.982310:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x6fe:0x0].0x0 since=36s v=(0 0) 00002000:00080000:3.0:1783520935.982312:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1794 count 1 00000100:00100000:2.0:1783520935.982318:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a83100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+487:3627:x1870152993175168:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.982318:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6fe:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.982328:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f0f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x6fe:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.982329:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025f16 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x702:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.982337:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1793 count 1 00010000:00010000:2.0:1783520935.982354:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025f1d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x701:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.982355:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000046d00412/0xd5401cbabc020ba9 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x702:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f13e2 expref: 521 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.982401:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x702:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.982405:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x702:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.982409:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025f16 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x702:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.982424:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000801985f0/0xd5401cbabc022a96 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6fe:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3186 expref: 521 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.982434:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000046d00412/0xd5401cbabc020ba9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x702:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13e2 expref: 521 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.982443:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002da19001/0xd5401cbabc020ba2 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x701:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f13d4 expref: 521 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.982449:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000801985f0/0xd5401cbabc022a96 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6fe:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3186 expref: 521 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.982471:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x701:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.982478:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x701:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.982480:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000046d00412/0xd5401cbabc020ba9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x702:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13e2 expref: 521 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.982484:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435080d0380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993175552:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.982487:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025f1d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x701:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.982504:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.982522:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435055cdf80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993175680:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.982542:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002da19001/0xd5401cbabc020ba2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x701:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13d4 expref: 521 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.982554:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002da19001/0xd5401cbabc020ba2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x701:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13d4 expref: 521 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.982557:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943515746cc0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.982561:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8c00 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.982568:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.982569:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511adaa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993175808:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.982603:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435080d3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993174144:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.982606:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a83480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993174400:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.982613:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435129679c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.982630:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ef3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.982632:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025ee5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.982633:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002429440 00000100:00100000:2.0:1783520935.982639:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002429440 00010000:00010000:2.0:1783520935.982658:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000df50a3e5/0xd5401cbabc020b86 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f138e expref: 522 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.982672:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.982683:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002429568 00000100:00100000:2.0:1783520935.982688:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002429568 00000100:00100000:0.0:1783520935.982690:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002429696 00000100:00100000:0.0:1783520935.982695:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002429696 00010000:00010000:2.0:1783520935.982703:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e3baad83/0xd5401cbabc020b6a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1364 expref: 524 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.982706:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36145 00010000:00010000:0.0:1783520935.982717:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009f4ae15d/0xd5401cbabc020b8d lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x6fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f139c expref: 524 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.982723:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e98a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+524:2388:x1870153002429440:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.982738:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36146 00010000:00010000:3.0:1783520935.982740:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.982748:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000df50a3e5/0xd5401cbabc020b86 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6fc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f138e expref: 524 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.982750:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+524:2388:x1870153002429568:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.982764:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36147 00010000:00010000:2.0:1783520935.982767:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.982774:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000df50a3e5/0xd5401cbabc020b86 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6fc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f138e expref: 524 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.982777:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e3baad83/0xd5401cbabc020b6a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6f9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1364 expref: 524 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.982777:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff0e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993174528:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.982778:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f3850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+524:2387:x1870153002429696:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.982793:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.982803:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009f4ae15d/0xd5401cbabc020b8d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6fd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f139c expref: 524 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.982804:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025efa lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6fd:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.982810:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e3baad83/0xd5401cbabc020b6a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6f9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1364 expref: 524 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.982834:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009f4ae15d/0xd5401cbabc020b8d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6fd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f139c expref: 524 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.982842:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e3baad83/0xd5401cbabc020b6a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6f9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1364 expref: 524 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.982854:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.982856:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009f4ae15d/0xd5401cbabc020b8d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6fd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f139c expref: 523 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.982858:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6f9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.982868:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000df50a3e5/0xd5401cbabc020b86 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6fc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f138e expref: 523 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.982870:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025ee5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.982874:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6fd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.982882:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6fd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.982890:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6fc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.982891:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025efa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.982893:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025ee5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.982897:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6fc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.982907:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ef3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.982914:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025efa lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.982925:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ef3 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.982929:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.982940:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.982943:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.982953:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025efa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.982956:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+521:2388:x1870153002429568:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 209us (389us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.982967:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f3850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+521:2387:x1870153002429696:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 192us (294us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.982970:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e98a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+521:2388:x1870153002429440:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 252us (483us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.982975:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36146 00010000:00010000:1.0:1783520935.982976:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025efa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.982988:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36147 00000100:00100000:3.0:1783520935.982996:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36145 00010000:00010000:1.0:1783520935.983000:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025efa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.983018:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025efa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.983019:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ef3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.983038:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ef3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.983045:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ef3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.983050:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ef3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.983054:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.983057:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025ee5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.983066:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520935.983069:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025ee5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.983080:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025ee5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.983087:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025ee5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.983089:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520935.983097:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025efa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.983105:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.983107:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.983111:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ef3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.983113:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025efa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.983134:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025efa lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.983149:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025efa lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.983156:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc025efa lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.983160:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a80380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+489:3627:x1870152993174272:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2648us (3016us total) trans 4294976056 rc 0/0 00000100:00100000:0.0:1783520935.983172:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d2a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+489:3626:x1870152993174016:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3338us (3402us total) trans 4294976057 rc 0/0 00010000:00010000:2.0:1783520935.983178:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.983182:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.983185:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.983185:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44556 00000100:00100000:0.0:1783520935.983193:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44555 00000004:00080000:0.0:1783520935.983227:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514d29180/2, rc 0, transno 4294976057 00010000:00010000:3.0:1783520935.983233:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.983239:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ef3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.983242:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993174016:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.983255:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ef3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.983271:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ef3 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.983280:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ef3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.983292:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993175936:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.983297:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.983300:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.983303:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.983325:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:3.0:1783520935.983326:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943509a81500/2, rc 0, transno 4294976056 00000100:00100000:3.0:1783520935.983338:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a81500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993174272:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.983353:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993175936 00000100:00100000:1.0:1783520935.983376:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993175936 00000100:00100000:1.0:1783520935.983412:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44562 00000100:00100000:1.0:1783520935.983421:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435080d3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+488:3625:x1870152993175936:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.983423:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.983427:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.983433:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025ee5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.983437:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1795 count 1 00010000:00010000:0.0:1783520935.983446:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.983449:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025ee5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.983452:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025f24 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x703:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.983462:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025ee5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.983474:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000babb5b69/0xd5401cbabc020bb7 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x703:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f13f0 expref: 518 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.983478:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025ee5 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.983488:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x703:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.983491:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc025ee5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.983505:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x703:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.983515:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025f24 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x703:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.983519:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435055cce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993176064:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.983528:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.983532:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a80e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+489:3627:x1870152993173376:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4392us (4497us total) trans 4294976058 rc 0/0 00010000:00010000:0.0:1783520935.983533:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.983538:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.983541:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44553 00010000:00010000:1.0:1783520935.983548:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000babb5b69/0xd5401cbabc020bb7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x703:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13f0 expref: 518 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.983555:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.983571:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000babb5b69/0xd5401cbabc020bb7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x703:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13f0 expref: 518 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.983576:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993176064 00000100:00100000:2.0:1783520935.983581:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993176064 00000100:00100000:1.0:1783520935.983585:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9b100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993176192:192.168.204.33@tcp:104: 00000004:00080000:3.0:1783520935.983604:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514ff2680/2, rc 0, transno 4294976058 00000100:00100000:2.0:1783520935.983611:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44563 00000100:00100000:3.0:1783520935.983619:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993173376:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.983623:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d29c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+489:3626:x1870152993176064:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.983638:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00002000:00080000:2.0:1783520935.983639:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1796 count 1 00010000:00010000:2.0:1783520935.983655:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025f2b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x704:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.983668:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b670e883/0xd5401cbabc020bbe lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x704:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f13fe expref: 518 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.983669:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435080d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993176320:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.983682:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x704:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.983686:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x704:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.983690:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025f2b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x704:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.983705:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.983715:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b670e883/0xd5401cbabc020bbe lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x704:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13fe expref: 518 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.983727:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993176320 00010000:00010000:2.0:1783520935.983728:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b670e883/0xd5401cbabc020bbe lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x704:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13fe expref: 518 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.983733:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993176320 00000100:00100000:2.0:1783520935.983744:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ad9880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993176448:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.983765:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44564 00000100:00100000:2.0:1783520935.983776:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967e40 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.983777:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d29880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+490:3627:x1870152993176320:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.983791:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1797 count 1 00010000:00010000:1.0:1783520935.983803:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025f32 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x705:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.983815:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e848f275/0xd5401cbabc020bd3 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x705:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f140c expref: 518 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.983821:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x705:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.983825:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x705:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.983828:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025f32 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x705:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.983849:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e848f275/0xd5401cbabc020bd3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x705:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f140c expref: 518 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.983865:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e848f275/0xd5401cbabc020bd3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x705:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f140c expref: 518 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.983876:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e98700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993176576:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.983904:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.983920:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9a680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993175424:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.983935:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025f01 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.984042:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435080d0380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993175552:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.984056:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f0f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x6fe:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.984076:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506228a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993175040:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.984095:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025f08 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x700:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.984102:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435055cdf80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993175680:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.984125:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025f16 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x702:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.984157:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511adaa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993175808:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.984170:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025f1d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x701:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.984482:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.984517:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002429824 00000100:00100000:1.0:1783520935.984523:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.984524:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002429824 00010000:00010000:2.0:1783520935.984544:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005b5ab580/0xd5401cbabc020b94 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13b8 expref: 519 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.984553:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.984584:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002429952 00000100:00100000:1.0:1783520935.984585:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002430080 00000100:00100000:2.0:1783520935.984589:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002429952 00000100:00100000:1.0:1783520935.984592:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002430080 00000100:00100000:3.0:1783520935.984597:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36148 00010000:00010000:2.0:1783520935.984605:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000801985f0/0xd5401cbabc022a96 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6fe:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3186 expref: 521 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.984611:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9a680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+521:2388:x1870153002429824:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.984616:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000010fcf996/0xd5401cbabc020b9b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x700:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13c6 expref: 521 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.984630:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.984638:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005b5ab580/0xd5401cbabc020b94 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x6ff:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13b8 expref: 521 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.984646:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36149 00010000:00010000:3.0:1783520935.984656:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005b5ab580/0xd5401cbabc020b94 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x6ff:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13b8 expref: 521 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.984657:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36150 00000100:00100000:2.0:1783520935.984661:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+521:2387:x1870153002429952:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520935.984674:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e98a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+521:2388:x1870153002430080:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.984678:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.984685:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.984686:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000801985f0/0xd5401cbabc022a96 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x6fe:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3186 expref: 521 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.984687:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005b5ab580/0xd5401cbabc020b94 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x6ff:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13b8 expref: 521 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.984694:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000010fcf996/0xd5401cbabc020b9b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x700:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13c6 expref: 521 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.984713:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000801985f0/0xd5401cbabc022a96 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6fe:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3186 expref: 521 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.984717:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ff:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.984726:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6ff:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.984737:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025f01 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.984742:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000010fcf996/0xd5401cbabc020b9b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x700:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13c6 expref: 520 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.984753:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000801985f0/0xd5401cbabc022a96 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6fe:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3186 expref: 520 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.984761:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000010fcf996/0xd5401cbabc020b9b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x700:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13c6 expref: 520 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.984768:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6fe:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.984774:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x6fe:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.984777:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x700:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.984779:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f0f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6fe:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.984783:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x700:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.984793:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025f08 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x700:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.984797:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f0f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6fe:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.984809:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025f01 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x6ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.984819:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025f08 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x700:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.984843:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.984848:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.984850:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520935.984873:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9a680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+518:2388:x1870153002429824:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 266us (390us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.984880:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e98a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+518:2388:x1870153002430080:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 209us (326us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.984883:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+518:2387:x1870153002429952:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 225us (360us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.984890:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36148 00010000:00010000:0.0:1783520935.984897:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f0f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.984901:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36150 00010000:00010000:0.0:1783520935.984910:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f0f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.984912:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36149 00010000:00010000:0.0:1783520935.984922:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f0f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.984927:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f0f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.984938:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.984948:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025f08 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x700:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.984948:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.984953:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f0f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.984970:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.984971:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025f08 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x700:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.984977:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.984977:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f0f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.984994:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025f08 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x700:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.984996:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f0f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.985004:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002430208 00000100:00100000:3.0:1783520935.985010:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002430208 00010000:00010000:2.0:1783520935.985018:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025f08 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x700:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.985025:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f0f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.985030:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.985039:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f0f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.985045:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.985055:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.985057:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.985060:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025f08 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x700:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.985065:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.985068:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025f08 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x700:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.985072:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000046d00412/0xd5401cbabc020ba9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x702:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13e2 expref: 516 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.985076:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025f08 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x700:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.985098:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025f08 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x700:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.985104:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc025f08 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x700:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.985113:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002430336 00000100:00100000:3.0:1783520935.985118:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002430336 00010000:00010000:0.0:1783520935.985118:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.985122:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.985125:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.985136:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002da19001/0xd5401cbabc020ba2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x701:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13d4 expref: 517 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.985161:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36151 00000100:00100000:1.0:1783520935.985167:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.985174:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+517:2387:x1870153002430208:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.985182:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350622b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+492:3626:x1870152993174912:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3235us (3319us total) trans 4294976059 rc 0/0 00000100:00100000:2.0:1783520935.985192:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44558 00000100:00100000:1.0:1783520935.985196:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002430464 00000100:00100000:1.0:1783520935.985201:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002430464 00000100:00100000:0.0:1783520935.985207:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d2bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+491:3628:x1870152993174656:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2968us (3453us total) trans 4294976060 rc 0/0 00000100:00100000:0.0:1783520935.985219:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44559 00010000:00010000:1.0:1783520935.985221:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000babb5b69/0xd5401cbabc020bb7 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x703:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13f0 expref: 518 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.985222:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025f01 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.985225:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.985235:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000046d00412/0xd5401cbabc020ba9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x702:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13e2 expref: 518 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.985245:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514ff1180/2, rc 0, transno 4294976060 00010000:00010000:2.0:1783520935.985248:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025f01 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.985257:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993174656:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.985269:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000046d00412/0xd5401cbabc020ba9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x702:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13e2 expref: 518 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.985272:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025f01 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x6ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.985290:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36152 00010000:00010000:2.0:1783520935.985294:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025f01 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.985304:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+518:2388:x1870153002430336:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.985305:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000046d00412/0xd5401cbabc020ba9 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x702:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13e2 expref: 518 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.985307:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.985309:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435080d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993176704:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.985323:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.985326:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x702:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.985331:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x702:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.985332:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002da19001/0xd5401cbabc020ba2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x701:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13d4 expref: 517 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.985339:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025f16 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x702:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.985352:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002da19001/0xd5401cbabc020ba2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x701:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13d4 expref: 517 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.985353:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.985355:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025f16 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x702:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.985399:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.985405:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002da19001/0xd5401cbabc020ba2 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x701:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13d4 expref: 517 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.985406:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025f01 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.985424:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993176704 00010000:00010000:3.0:1783520935.985427:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbba00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520935.985430:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993176704 00010000:00010000:1.0:1783520935.985431:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x701:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.985433:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025f01 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x6ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.985437:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x701:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.985442:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025f1d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x701:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.985450:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025f1d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x701:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.985465:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514ff1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+492:3625:x1870152993174784:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3523us (3644us total) trans 4294976061 rc 0/0 00000004:00080000:0.0:1783520935.985470:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514ff0000/2, rc 0, transno 4294976061 00010000:00010000:1.0:1783520935.985472:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.985475:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44557 00000100:00100000:0.0:1783520935.985483:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993174784:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.985512:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44565 00000100:00100000:1.0:1783520935.985520:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+516:2388:x1870153002430336:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 218us (492us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.985524:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025f01 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x6ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.985530:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435080d0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+491:3628:x1870152993176704:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.985536:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36152 00000100:00100000:0.0:1783520935.985541:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435080d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993176832:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.985549:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025f01 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x6ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.985554:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1798 count 1 00000100:00100000:1.0:1783520935.985562:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36153 00010000:00010000:3.0:1783520935.985563:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025f01 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x6ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.985573:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025f39 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x706:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.985579:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575c700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+515:2387:x1870153002430464:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.985581:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.985582:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.985586:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.985591:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.985595:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.985600:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000031d73f44/0xd5401cbabc022985 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x706:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3075 expref: 515 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.985605:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000babb5b69/0xd5401cbabc020bb7 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x703:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13f0 expref: 515 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.985615:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x706:0x0].0x0 since=36s v=(0 0) 00010000:00010000:2.0:1783520935.985622:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x706:0x0].0x0 since=0s v=(1 0) 00000100:00100000:0.0:1783520935.985623:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993176832 00010000:00010000:1.0:1783520935.985628:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000babb5b69/0xd5401cbabc020bb7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x703:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13f0 expref: 515 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.985630:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993176832 00000004:00080000:3.0:1783520935.985631:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514ff1880/2, rc 0, transno 4294976059 00010000:00010000:2.0:1783520935.985631:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025f39 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x706:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.985653:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993174912:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.985656:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x703:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.985662:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x703:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.985666:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025f24 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x703:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.985670:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000031d73f44/0xd5401cbabc022985 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x706:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3075 expref: 515 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.985683:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.985688:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000031d73f44/0xd5401cbabc022985 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x706:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3075 expref: 515 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.985688:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025f24 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x703:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.985698:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1783520935.985711:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+515:2387:x1870153002430208:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 537us (735us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.985713:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506d9f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993177088:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.985717:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.985718:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511adaa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993176960:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.985733:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36151 00000100:00100000:1.0:1783520935.985745:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575c700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+514:2387:x1870153002430464:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 172us (580us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.985754:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36153 00000100:00100000:0.0:1783520935.985767:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.985778:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967a80 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.985802:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993177088 00000100:00100000:2.0:1783520935.985806:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993177088 00010000:00010000:0.0:1783520935.985811:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025f16 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x702:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.985812:0:12734:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000babb5b69/0xd5401cbabc020bb7 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x703:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13f0 expref: 513 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.985833:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44566 00010000:00010000:0.0:1783520935.985834:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025f16 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x702:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.985841:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9b100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993176192:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.985847:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435055cdf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+493:3625:x1870152993176832:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.985853:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025f16 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x702:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.985862:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1799 count 1 00010000:00010000:1.0:1783520935.985864:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025f24 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.985872:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025f16 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x702:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.985879:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025f40 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x707:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.985880:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025f24 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.985887:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.985904:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.985906:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025f24 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.985911:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000aeb466bf/0xd5401cbabc020bcc lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x707:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1428 expref: 512 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.985916:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.985923:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025f24 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.985924:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025f16 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x702:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.985927:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x707:0x0].0x0 since=37s v=(0 0) 00000100:00100000:3.0:1783520935.985935:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.985938:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x707:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.985949:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025f40 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x707:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.985951:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.985952:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025f24 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.985960:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025f16 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x702:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.985979:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000aeb466bf/0xd5401cbabc020bcc lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x707:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1428 expref: 512 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.985981:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025f16 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x702:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.985989:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520935.986000:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002430592 00000100:00100000:3.0:1783520935.986006:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002430592 00010000:00010000:2.0:1783520935.986007:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000aeb466bf/0xd5401cbabc020bcc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x707:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1428 expref: 512 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.986008:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025f16 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x702:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.986025:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b670e883/0xd5401cbabc020bbe lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x704:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13fe expref: 513 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.986034:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025f16 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x702:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.986037:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ada680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993177216:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.986053:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.986059:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025f24 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.986060:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.986063:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.986068:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.986072:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.986078:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025f24 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.986079:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435080d3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+494:3625:x1870152993175936:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2658us (2753us total) trans 4294976062 rc 0/0 00000004:00080000:0.0:1783520935.986094:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e9b800/2, rc 0, transno 4294976062 00010000:00010000:2.0:1783520935.986098:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025f24 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.986098:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44562 00000100:00100000:3.0:1783520935.986106:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002430720 00000100:00100000:0.0:1783520935.986109:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993175936:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.986112:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002430720 00010000:00010000:2.0:1783520935.986115:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025f24 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.986123:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44567 00010000:00010000:3.0:1783520935.986131:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e848f275/0xd5401cbabc020bd3 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x705:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f140c expref: 514 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.986132:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc025f24 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.986139:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c9a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+493:3625:x1870152993177088:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.986159:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1800 count 1 00000100:00100000:0.0:1783520935.986167:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:2.0:1783520935.986168:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.986172:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.986172:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025f47 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x708:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.986178:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.986182:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502c99180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993177344:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.986191:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36154 00010000:00010000:1.0:1783520935.986191:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001f26b89b/0xd5401cbabc020bda lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x708:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1436 expref: 514 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.986204:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36155 00000100:00100000:3.0:1783520935.986208:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f3050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+514:2389:x1870153002430592:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.986219:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.986221:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d29f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+514:2389:x1870153002430720:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.986222:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x708:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.986223:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.986231:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x708:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.986233:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b670e883/0xd5401cbabc020bbe lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x704:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13fe expref: 514 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.986237:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.986237:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025f47 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x708:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.986247:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e848f275/0xd5401cbabc020bd3 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x705:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f140c expref: 514 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.986256:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b670e883/0xd5401cbabc020bbe lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x704:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13fe expref: 514 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.986278:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e848f275/0xd5401cbabc020bd3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x705:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f140c expref: 514 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.986278:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001f26b89b/0xd5401cbabc020bda lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x708:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1436 expref: 514 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.986283:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x704:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.986292:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x704:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.986303:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025f2b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x704:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.986309:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001f26b89b/0xd5401cbabc020bda lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x708:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1436 expref: 514 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.986311:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x705:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.986322:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x705:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.986322:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435080d1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+494:3628:x1870152993175296:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4030us (4138us total) trans 4294976063 rc 0/0 00010000:00010000:2.0:1783520935.986334:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025f32 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x705:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.986341:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575ea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993177472:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.986343:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44560 00010000:00010000:2.0:1783520935.986354:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025f32 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x705:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.986427:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3eb40 going to sleep for 11 seconds 00000004:00080000:0.0:1783520935.986429:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514ff3800/2, rc 0, transno 4294976063 00010000:00010000:2.0:1783520935.986440:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.986442:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993175296:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.986448:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025f2b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x704:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.986468:0:35274:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e848f275/0xd5401cbabc020bd3 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x705:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f140c expref: 514 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.986470:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.986474:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d29f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+514:2389:x1870153002430720:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 255us (541us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.986490:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36155 00000100:00100000:1.0:1783520935.986507:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e98700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993176576:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.986511:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502c98380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993177600:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.986527:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025f32 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x705:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.986538:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025f32 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x705:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.986545:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f3050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+512:2389:x1870153002430592:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 339us (660us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520935.986547:0:8410:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b670e883/0xd5401cbabc020bbe lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x704:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f13fe expref: 512 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.986549:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520935.986573:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025f32 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x705:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.986575:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ad9880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993176448:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.986577:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993177344 00000100:00100000:0.0:1783520935.986584:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993177344 00010000:00010000:1.0:1783520935.986592:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025f32 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x705:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.986599:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025f2b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.986601:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025f32 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x705:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.986605:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993177600 00000100:00100000:0.0:1783520935.986610:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993177600 00000100:00100000:3.0:1783520935.986619:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36154 00010000:00010000:2.0:1783520935.986626:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025f2b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.986627:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44568 00010000:00010000:2.0:1783520935.986638:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025f2b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.986640:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.986642:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c99500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+495:3628:x1870152993177344:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.986658:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025f2b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.986659:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.986669:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025f32 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x705:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.986670:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025f2b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.986690:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.986695:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025f32 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x705:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.986705:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.986706:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025f32 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x705:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.986712:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.986721:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025f2b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.986727:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025f32 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x705:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.986733:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.986739:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025f2b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.986740:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d29880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+496:3627:x1870152993176320:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2966us (3035us total) trans 4294976064 rc 0/0 00010000:00010000:0.0:1783520935.986742:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc025f32 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x705:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.986759:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44564 00010000:00010000:3.0:1783520935.986765:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025f2b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.986780:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.986782:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025f2b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.986785:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.986787:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44569 00010000:00010000:0.0:1783520935.986789:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.986799:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d29c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+496:3626:x1870152993176064:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3178us (3242us total) trans 4294976065 rc 0/0 00010000:00010000:3.0:1783520935.986800:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc025f2b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.986802:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c9b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+496:3628:x1870152993177600:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.986807:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1801 count 1 00000100:00100000:2.0:1783520935.986819:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44563 00002000:00080000:1.0:1783520935.986823:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1802 count 1 00010000:00010000:0.0:1783520935.986826:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025f4e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x709:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.986832:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.986837:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.986839:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025f55 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x70a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.986841:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000aa54acfd/0xd5401cbabc020be1 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x709:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1444 expref: 510 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.986842:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.986863:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435129679c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.986863:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x709:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.986866:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002a28d518/0xd5401cbabc020be8 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x70a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1452 expref: 510 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.986870:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435080d1c00/2, rc 0, transno 4294976064 00010000:00010000:0.0:1783520935.986871:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x709:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.986881:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025f4e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x709:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.986884:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025f1d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x701:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.986884:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x70a:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.986894:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x70a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.986905:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025f55 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x70a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.986908:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025f1d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x701:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.986924:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025f1d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x701:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.986931:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025f1d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x701:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.986937:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435080d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993176320:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.986942:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002a28d518/0xd5401cbabc020be8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x70a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1452 expref: 510 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.986945:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520935.986950:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502c9b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993177728:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.986963:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002a28d518/0xd5401cbabc020be8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x70a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1452 expref: 510 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.986970:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435055cce00/2, rc 0, transno 4294976065 00000100:00100000:1.0:1783520935.986978:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575ce00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993177856:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.986984:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435055cce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993176064:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.986991:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.987011:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e240 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.987045:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a83100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+496:3627:x1870152993175168:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4732us (4824us total) trans 4294976066 rc 0/0 00000004:00080000:3.0:1783520935.987051:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514ff3480/2, rc 0, transno 4294976066 00000100:00100000:2.0:1783520935.987053:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44561 00000100:00100000:3.0:1783520935.987059:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993175168:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.987072:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993177728 00000100:00100000:3.0:1783520935.987078:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502c98a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993177984:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.987079:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993177728 00010000:00010000:0.0:1783520935.987092:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.987099:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025f1d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x701:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.987114:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44570 00000100:00100000:3.0:1783520935.987115:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.987120:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.987125:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025f1d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x701:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.987127:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c99c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+496:3625:x1870152993177728:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.987142:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025f1d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x701:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.987146:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1803 count 1 00000100:00100000:3.0:1783520935.987152:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cbaaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993178112:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.987161:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025f1d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x701:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.987163:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025f5c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x70b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.987185:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.987185:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025f1d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x701:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.987192:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000027b98804/0xd5401cbabc020bef lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x70b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1460 expref: 510 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.987204:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.987207:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.987210:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x70b:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.987211:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.987216:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x70b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.987222:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025f5c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x70b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.987244:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000027b98804/0xd5401cbabc020bef lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x70b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1460 expref: 510 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.987251:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000027b98804/0xd5401cbabc020bef lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x70b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1460 expref: 510 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.987258:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000aa54acfd/0xd5401cbabc020be1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x709:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1444 expref: 510 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.987263:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ad9880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993178240:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.987267:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.987282:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000aa54acfd/0xd5401cbabc020be1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x709:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1444 expref: 510 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.987295:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435122b6f00 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.987295:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993178368:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.987307:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993177984 00000100:00100000:2.0:1783520935.987311:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993177984 00000100:00100000:1.0:1783520935.987319:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002430848 00000100:00100000:1.0:1783520935.987325:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002430848 00000100:00100000:2.0:1783520935.987331:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993178112 00000100:00100000:0.0:1783520935.987331:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435192190c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.987335:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993178112 00010000:00010000:1.0:1783520935.987349:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000031d73f44/0xd5401cbabc022985 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x706:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3075 expref: 511 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.987381:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44571 00000100:00100000:2.0:1783520935.987391:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44572 00000100:00100000:0.0:1783520935.987396:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+498:3626:x1870152993177984:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.987405:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a83480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+498:3627:x1870152993178112:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.987410:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36156 00002000:00080000:0.0:1783520935.987410:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1804 count 1 00000100:00100000:1.0:1783520935.987421:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94360550f100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+511:2388:x1870153002430848:12345-192.168.204.33@tcp:103:ldlm_bl.0 00002000:00080000:2.0:1783520935.987422:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1805 count 1 00010000:00010000:0.0:1783520935.987426:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025f63 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x70c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.987435:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.987436:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025f6a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x70d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.987442:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000031d73f44/0xd5401cbabc022985 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x706:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3075 expref: 511 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.987442:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005ac70d48/0xd5401cbabc020ce4 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x70c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f146e expref: 511 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.987452:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b2d23d8c/0xd5401cbabc020bf6 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x70d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f147c expref: 511 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.987469:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x70c:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.987475:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000031d73f44/0xd5401cbabc022985 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x706:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3075 expref: 511 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.987479:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x70c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.987481:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x70d:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.987489:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x70d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.987489:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025f63 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x70c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.987509:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025f6a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x70d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.987513:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x706:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.987523:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x706:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.987529:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025f39 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x706:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.987536:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005ac70d48/0xd5401cbabc020ce4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x70c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f146e expref: 511 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.987539:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025f39 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x706:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.987542:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b2d23d8c/0xd5401cbabc020bf6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x70d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f147c expref: 511 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.987565:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005ac70d48/0xd5401cbabc020ce4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x70c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f146e expref: 511 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.987569:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b2d23d8c/0xd5401cbabc020bf6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x70d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f147c expref: 511 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.987573:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.987593:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511adbb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993178624:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.987593:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cbab100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993178496:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.987603:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94360550f100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+511:2388:x1870153002430848:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 181us (338us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.987613:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36156 00000100:00100000:2.0:1783520935.987645:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967e40 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.987645:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c016a80 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.987692:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000031d73f44/0xd5401cbabc022985 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x706:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3075 expref: 510 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.987703:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511adaa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993176960:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.987716:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025f39 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x706:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.987724:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025f39 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x706:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.987736:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025f39 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x706:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.987744:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025f39 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x706:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.987752:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025f39 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x706:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.987770:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.987783:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.987789:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025f39 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x706:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.987803:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.987808:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025f39 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x706:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.987816:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025f39 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x706:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.987827:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025f39 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x706:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.987834:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc025f39 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x706:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.987849:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.987852:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.987855:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.987861:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435080d0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+499:3628:x1870152993176704:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2337us (2510us total) trans 4294976067 rc 0/0 00000100:00100000:2.0:1783520935.987869:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44565 00000004:00080000:1.0:1783520935.987912:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435080d2680/2, rc 0, transno 4294976067 00000100:00100000:1.0:1783520935.987921:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435080d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993176704:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.987977:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cbab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993178752:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.988004:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.988030:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993178752 00000100:00100000:2.0:1783520935.988034:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993178752 00000100:00100000:2.0:1783520935.988057:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44573 00000100:00100000:2.0:1783520935.988061:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+499:3628:x1870152993178752:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.988071:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1806 count 1 00010000:00010000:2.0:1783520935.988084:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025f71 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x70e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.988096:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b4138498/0xd5401cbabc022874 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x70e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f64 expref: 509 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.988103:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x70e:0x0].0x0 since=36s v=(0 0) 00010000:00010000:2.0:1783520935.988106:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x70e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.988110:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025f71 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x70e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.988124:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b4138498/0xd5401cbabc022874 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x70e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f64 expref: 509 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.988130:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b4138498/0xd5401cbabc022874 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x70e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f64 expref: 509 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.988137:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511adaa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993178880:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.988163:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967a80 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.989304:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.989331:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575ea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993177472:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.989351:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025f47 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x708:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.989387:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.989388:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002430976 00000100:00100000:2.0:1783520935.989395:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002430976 00000100:00100000:1.0:1783520935.989418:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002431104 00000100:00100000:1.0:1783520935.989424:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002431104 00010000:00010000:2.0:1783520935.989426:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001f26b89b/0xd5401cbabc020bda lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x708:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1436 expref: 510 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.989445:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000aeb466bf/0xd5401cbabc020bcc lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x707:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1428 expref: 511 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.989461:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36157 00000100:00100000:0.0:1783520935.989466:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.989473:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36158 00000100:00100000:2.0:1783520935.989477:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cba9180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+511:2387:x1870153002430976:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520935.989485:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002431360 00010000:00010000:2.0:1783520935.989489:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.989489:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5bb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+511:2388:x1870153002431104:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520935.989502:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002431360 00010000:00010000:2.0:1783520935.989513:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001f26b89b/0xd5401cbabc020bda lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x708:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1436 expref: 512 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.989513:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.989529:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001f26b89b/0xd5401cbabc020bda lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x708:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1436 expref: 512 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.989530:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.989548:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000aeb466bf/0xd5401cbabc020bcc lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x707:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1428 expref: 512 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.989551:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.989563:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001f26b89b/0xd5401cbabc020bda lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x708:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1436 expref: 512 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.989567:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000aeb466bf/0xd5401cbabc020bcc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x707:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1428 expref: 512 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.989586:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x708:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.989587:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993178368:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.989592:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x708:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.989593:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x707:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.989602:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x707:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.989603:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025f47 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x708:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.989614:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025f47 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x708:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.989620:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025f40 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x707:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.989622:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025f4e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x709:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.989631:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000027b98804/0xd5401cbabc020bef lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x70b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1460 expref: 511 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.989641:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.989650:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025f40 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x707:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.989668:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.989675:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cba9180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+511:2387:x1870153002430976:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 202us (373us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.989683:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36157 00000100:00100000:1.0:1783520935.989694:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5bb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+511:2388:x1870153002431104:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 208us (306us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.989701:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002431232 00000100:00100000:2.0:1783520935.989706:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002431232 00000100:00100000:1.0:1783520935.989706:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36158 00010000:00010000:2.0:1783520935.989722:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002a28d518/0xd5401cbabc020be8 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x70a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1452 expref: 511 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.989736:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002431488 00000100:00100000:2.0:1783520935.989739:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002431488 00010000:00010000:2.0:1783520935.989750:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000aa54acfd/0xd5401cbabc020be1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x709:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1444 expref: 512 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.989763:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36159 00000100:00100000:2.0:1783520935.989771:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508059f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+511:2387:x1870153002431232:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.989779:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520935.989780:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36160 00010000:00010000:2.0:1783520935.989788:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002a28d518/0xd5401cbabc020be8 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x70a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1452 expref: 511 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.989793:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f3450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+511:2387:x1870153002431488:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.989808:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.989815:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000aa54acfd/0xd5401cbabc020be1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x709:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1444 expref: 511 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.989822:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002a28d518/0xd5401cbabc020be8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x70a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1452 expref: 511 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.989854:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x70a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.989858:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x70a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.989862:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025f55 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.989870:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025f55 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x70a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.989875:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000aa54acfd/0xd5401cbabc020be1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x709:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1444 expref: 511 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.989891:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.989892:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000aa54acfd/0xd5401cbabc020be1 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x709:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1444 expref: 511 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.989913:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x709:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.989916:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508059f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+510:2387:x1870153002431232:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 143us (405us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.989917:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x709:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.989923:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025f4e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x709:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.989927:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36161 00010000:00010000:1.0:1783520935.989935:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025f4e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x709:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.989940:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36159 00000100:00100000:3.0:1783520935.989944:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5bad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+510:2388:x1870153002431360:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.989958:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.989963:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000027b98804/0xd5401cbabc020bef lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x70b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1460 expref: 509 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.989968:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.989978:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000027b98804/0xd5401cbabc020bef lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x70b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1460 expref: 509 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.989994:0:178496:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000aeb466bf/0xd5401cbabc020bcc lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x707:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1428 expref: 509 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.989995:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025f4e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x709:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.990002:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f3450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+509:2387:x1870153002431488:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 210us (453us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.990007:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x70b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.990018:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x70b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.990021:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025f4e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x709:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.990024:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ada680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993177216:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.990024:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36160 00010000:00010000:3.0:1783520935.990028:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025f5c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.990048:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025f40 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x707:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.990051:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025f5c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x70b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.990051:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025f4e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x709:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.990062:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025f47 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x708:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.990072:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025f40 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x707:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.990081:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025f4e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x709:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.990086:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.990089:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025f47 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x708:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.990097:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025f40 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x707:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.990114:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5bad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+507:2388:x1870153002431360:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 172us (649us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.990116:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025f47 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x708:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.990117:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.990119:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025f40 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x707:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.990131:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:3.0:1783520935.990134:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36161 00010000:00010000:0.0:1783520935.990138:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025f4e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x709:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.990143:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025f47 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x708:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.990148:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025f40 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x707:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.990160:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.990166:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025f4e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x709:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.990176:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025f4e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x709:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.990185:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:0.0:1783520935.990185:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025f4e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x709:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.990192:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520935.990194:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520935.990202:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025f47 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x708:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.990204:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025f4e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x709:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.990224:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbbe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.990230:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025f47 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x708:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.990232:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.990238:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025f40 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x707:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.990244:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025f47 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x708:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.990245:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.990250:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025f40 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x707:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.990262:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025f40 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x707:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.990265:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025f47 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x708:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.990270:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025f40 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x707:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.990290:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc025f40 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x707:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.990293:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025f47 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x708:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.990305:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c9a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+501:3625:x1870152993177088:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4167us (4537us total) trans 4294976068 rc 0/0 00010000:00010000:0.0:1783520935.990306:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.990308:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435055cdf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+501:3625:x1870152993176832:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4462us (4730us total) trans 4294976069 rc 0/0 00010000:00010000:0.0:1783520935.990310:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.990313:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.990320:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.990325:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44567 00010000:00010000:3.0:1783520935.990332:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.990332:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44566 00010000:00010000:3.0:1783520935.990337:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.990342:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520935.990347:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506d9f800/2, rc 0, transno 4294976068 00000100:00100000:0.0:1783520935.990390:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506d9f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993177088:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.990391:0:35274:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002a28d518/0xd5401cbabc020be8 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x70a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1452 expref: 506 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.990404:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435080d1500/2, rc 0, transno 4294976069 00000100:00100000:1.0:1783520935.990409:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575ce00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993177856:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.990410:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435080d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993176832:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.990428:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000027b98804/0xd5401cbabc020bef lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x70b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1460 expref: 505 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.990433:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025f55 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.990444:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ad9880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993178240:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.990446:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025f55 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.990457:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025f55 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.990467:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025f5c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.990468:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025f55 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.990488:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:1.0:1783520935.990502:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025f55 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x70a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.990503:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025f5c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.990514:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993179136:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.990529:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025f5c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.990538:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025f5c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.990541:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.990547:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:3.0:1783520935.990552:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.990558:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025f5c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x70b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.990558:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025f55 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x70a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.990579:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508059f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993179008:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.990587:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.990592:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.990598:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025f55 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x70a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.990609:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.990609:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025f55 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x70a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.990624:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025f55 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x70a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.990629:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc025f55 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x70a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.990640:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.990642:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025f5c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x70b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.990646:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.990648:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025f5c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x70b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.990653:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025f5c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x70b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.990658:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025f5c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x70b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.990662:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc025f5c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x70b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.990670:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.990672:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.990673:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.990678:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.990753:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c9b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+502:3628:x1870152993177600:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3953us (4207us total) trans 4294976070 rc 0/0 00000100:00100000:0.0:1783520935.990762:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c99500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+502:3628:x1870152993177344:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4125us (4547us total) trans 4294976072 rc 0/0 00000100:00100000:1.0:1783520935.990766:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44569 00000100:00100000:2.0:1783520935.990769:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c99c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+502:3625:x1870152993177728:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3642us (3779us total) trans 4294976071 rc 0/0 00000100:00100000:3.0:1783520935.990771:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520935.990779:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44568 00000100:00100000:2.0:1783520935.990786:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44570 00000100:00100000:2.0:1783520935.990809:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993179136 00000004:00080000:1.0:1783520935.990813:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502c98380/2, rc 0, transno 4294976070 00000100:00100000:2.0:1783520935.990814:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993179136 00000100:00100000:3.0:1783520935.990822:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000004:00080000:0.0:1783520935.990823:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502c9b480/2, rc 0, transno 4294976071 00000100:00100000:1.0:1783520935.990829:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502c98380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993177600:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.990833:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502c9b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993177728:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.990845:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993179008 00000100:00100000:2.0:1783520935.990850:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993179008 00000004:00080000:1.0:1783520935.990851:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502c99180/2, rc 0, transno 4294976072 00000100:00100000:1.0:1783520935.990858:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502c99180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993177344:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.990869:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1783520935.990877:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44574 00000100:00100000:1.0:1783520935.990880:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cba8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993179264:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.990891:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94360550f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+501:3626:x1870152993179136:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.990899:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00002000:00080000:2.0:1783520935.990910:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1808 count 1 00000100:00100000:3.0:1783520935.990913:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d321c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993179392:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.990918:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993179520:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.990922:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.990929:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025f78 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x710:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.990943:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000bbdbe055/0xd5401cbabc020c0b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x710:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f14a6 expref: 504 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.990957:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x710:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.990961:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x710:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.990962:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993179264 00000100:00100000:1.0:1783520935.990968:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993179264 00000100:00100000:0.0:1783520935.990968:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.990969:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.990972:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025f78 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x710:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.990987:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993179520 00000100:00100000:1.0:1783520935.990990:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993179520 00010000:00010000:2.0:1783520935.990999:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000bbdbe055/0xd5401cbabc020c0b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x710:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14a6 expref: 504 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.991005:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993179392 00000100:00100000:1.0:1783520935.991009:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993179392 00000100:00100000:0.0:1783520935.991016:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44575 00010000:00010000:2.0:1783520935.991017:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000bbdbe055/0xd5401cbabc020c0b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x710:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14a6 expref: 504 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.991031:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94360550c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+504:3626:x1870152993179008:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.991034:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44576 00000100:00100000:2.0:1783520935.991035:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ad9880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993179648:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.991045:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+504:3628:x1870152993179264:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.991051:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1807 count 1 00002000:00080000:1.0:1783520935.991057:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1809 count 1 00010000:00010000:0.0:1783520935.991066:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f7f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x70f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.991070:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025f86 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x711:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.991077:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff9436298a6480 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.991092:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004cb354b0/0xd5401cbabc020c12 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x711:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f14b4 expref: 504 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.991093:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002431616 00010000:00010000:0.0:1783520935.991093:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000007d16a6a/0xd5401cbabc020c04 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x70f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1498 expref: 504 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.991099:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002431616 00010000:00010000:2.0:1783520935.991115:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b2d23d8c/0xd5401cbabc020bf6 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x70d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f147c expref: 505 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.991116:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x70f:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.991124:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x70f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.991133:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f7f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x70f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.991134:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x711:0x0].0x0 since=37s v=(0 0) 00000100:00100000:2.0:1783520935.991143:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002431744 00010000:00010000:1.0:1783520935.991143:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x711:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.991147:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.991151:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025f86 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x711:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.991153:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002431744 00010000:00010000:0.0:1783520935.991158:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000007d16a6a/0xd5401cbabc020c04 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x70f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1498 expref: 506 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.991171:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005ac70d48/0xd5401cbabc020ce4 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x70c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f146e expref: 506 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.991175:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000007d16a6a/0xd5401cbabc020c04 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x70f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1498 expref: 506 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.991186:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004cb354b0/0xd5401cbabc020c12 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x711:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14b4 expref: 506 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.991198:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11f800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993179776:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.991201:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004cb354b0/0xd5401cbabc020c12 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x711:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14b4 expref: 506 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.991204:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002431872 00000100:00100000:2.0:1783520935.991211:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002431872 00000100:00100000:1.0:1783520935.991215:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575ea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993179904:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.991230:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c016cc0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520935.991236:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b4138498/0xd5401cbabc022874 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x70e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f64 expref: 507 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.991249:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e3c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.991253:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44577 00000100:00100000:0.0:1783520935.991265:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+504:3625:x1870152993179520:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.991270:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36162 00000100:00100000:1.0:1783520935.991280:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36163 00002000:00080000:0.0:1783520935.991281:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1811 count 1 00000100:00100000:2.0:1783520935.991286:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94360550d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+507:2389:x1870153002431616:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520935.991293:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36164 00000100:00100000:1.0:1783520935.991294:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94360550c000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+507:2389:x1870153002431744:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.991295:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025f8d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x713:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.991299:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.991309:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.991310:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b2d23d8c/0xd5401cbabc020bf6 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x70d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f147c expref: 507 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.991311:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d29f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+507:2387:x1870153002431872:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.991312:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000303d326a/0xd5401cbabc020c20 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x713:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f14d0 expref: 507 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.991317:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005ac70d48/0xd5401cbabc020ce4 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x70c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f146e expref: 507 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.991331:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.991343:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b2d23d8c/0xd5401cbabc020bf6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x70d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f147c expref: 507 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.991344:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b4138498/0xd5401cbabc022874 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x70e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f64 expref: 507 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.991348:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005ac70d48/0xd5401cbabc020ce4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x70c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f146e expref: 507 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.991354:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x713:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.991376:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x713:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.991386:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025f8d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x713:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.991400:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x70d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.991402:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b4138498/0xd5401cbabc022874 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x70e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f64 expref: 507 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.991404:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x70d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.991408:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x70c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.991411:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000303d326a/0xd5401cbabc020c20 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x713:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14d0 expref: 507 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.991412:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025f6a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.991418:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x70c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.991432:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025f63 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.991436:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025f6a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x70d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.991439:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x70e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.991440:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000303d326a/0xd5401cbabc020c20 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x713:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14d0 expref: 507 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.991450:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x70e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520935.991456:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025f63 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x70c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.991463:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025f71 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.991468:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.991475:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11d180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993180032:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.991487:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94360550d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+507:2389:x1870153002431616:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 207us (722us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.991489:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025f71 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x70e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.991490:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.991510:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36162 00000100:00100000:1.0:1783520935.991532:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94360550c000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+506:2389:x1870153002431744:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 240us (708us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520935.991540:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c016240 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.991541:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36163 00010000:00010000:2.0:1783520935.991550:0:8410:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b2d23d8c/0xd5401cbabc020bf6 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x70d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f147c expref: 506 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.991563:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511adbb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993178624:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.991576:0:35275:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005ac70d48/0xd5401cbabc020ce4 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x70c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f146e expref: 504 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.991580:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025f6a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.991583:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44578 00010000:00010000:3.0:1783520935.991595:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.991599:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635787450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+504:3627:x1870152993179392:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.991602:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025f6a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.991606:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cbab100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993178496:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.991611:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025f6a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.991621:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025f6a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.991627:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025f6a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x70d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.991627:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025f63 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520935.991631:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1810 count 1 00000100:00100000:3.0:1783520935.991634:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d29f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+503:2387:x1870153002431872:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 325us (536us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.991649:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025f94 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x712:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.991656:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520935.991660:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36164 00010000:00010000:0.0:1783520935.991663:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.991671:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025f6a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x70d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.991679:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000137d5946/0xd5401cbabc020c19 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x712:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f14c2 expref: 502 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.991700:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x712:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.991701:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.991706:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x712:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.991708:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025f6a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x70d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.991711:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025f94 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x712:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.991724:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025f6a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x70d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.991736:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025f6a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x70d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.991741:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc025f6a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x70d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.991745:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a83480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+505:3627:x1870152993178112:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4342us (4561us total) trans 4294976073 rc 0/0 00010000:00010000:1.0:1783520935.991745:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000137d5946/0xd5401cbabc020c19 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x712:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14c2 expref: 502 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.991762:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.991765:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44572 00010000:00010000:0.0:1783520935.991766:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.991769:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.991776:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025f63 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.991781:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000137d5946/0xd5401cbabc020c19 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x712:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14c2 expref: 502 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.991785:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025f63 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.991803:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575df80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993180160:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.991803:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025f63 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.991822:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025f63 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x70c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520935.991834:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cbaaa00/2, rc 0, transno 4294976073 00010000:00010000:0.0:1783520935.991837:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.991840:0:9151:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b4138498/0xd5401cbabc022874 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x70e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f64 expref: 502 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.991844:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e240 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.991854:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cbaaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993178112:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.991859:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511adaa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993178880:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.991877:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.991880:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025f71 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.991883:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025f63 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x70c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.991892:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025f71 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.991892:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:3.0:1783520935.991899:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbebe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.991902:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025f71 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.991905:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993180288:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.991906:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025f63 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x70c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.991917:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025f71 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.991923:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025f63 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x70c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.991924:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025f71 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x70e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.991944:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.991959:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.991961:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025f63 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x70c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.991970:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc025f63 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x70c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.991970:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.991976:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025f71 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x70e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.991986:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.991989:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025f71 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x70e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.991993:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.991995:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025f71 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x70e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.992007:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025f71 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x70e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.992015:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc025f71 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x70e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.992032:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.992035:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.992039:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.992050:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+505:3628:x1870152993178752:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3986us (4043us total) trans 4294976074 rc 0/0 00010000:00010000:3.0:1783520935.992051:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.992058:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520935.992061:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cbab800/2, rc 0, transno 4294976074 00000100:00100000:2.0:1783520935.992062:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44573 00000100:00100000:2.0:1783520935.992079:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993180288 00000100:00100000:1.0:1783520935.992082:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cbab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993178752:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.992085:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993180288 00000100:00100000:2.0:1783520935.992116:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44579 00000100:00100000:2.0:1783520935.992123:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+505:3625:x1870152993180288:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.992127:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cbab100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993180416:0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.992136:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1812 count 1 00010000:00010000:2.0:1783520935.992149:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025f9b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x714:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.992149:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.992168:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000055b02f97/0xd5401cbabc020c27 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x714:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f14de expref: 501 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.992172:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.992180:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x714:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.992184:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x714:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.992189:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025f9b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x714:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.992190:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514ff0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+506:3626:x1870152993177984:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4798us (5079us total) trans 4294976075 rc 0/0 00000100:00100000:0.0:1783520935.992207:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44571 00000004:00080000:3.0:1783520935.992215:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502c98a80/2, rc 0, transno 4294976075 00010000:00010000:2.0:1783520935.992220:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000055b02f97/0xd5401cbabc020c27 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x714:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14de expref: 501 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.992226:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993180416 00000100:00100000:3.0:1783520935.992230:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502c98a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993177984:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.992232:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993180416 00010000:00010000:2.0:1783520935.992235:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000055b02f97/0xd5401cbabc020c27 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x714:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14de expref: 501 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.992251:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511adaa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993180544:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.992257:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44580 00000100:00100000:0.0:1783520935.992268:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5bb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+506:3627:x1870152993180416:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.992281:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1813 count 1 00000100:00100000:2.0:1783520935.992285:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435122b6f00 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.992294:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993180672:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.992296:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025fa2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x715:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.992302:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002432000 00000100:00100000:3.0:1783520935.992308:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002432000 00010000:00010000:0.0:1783520935.992314:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009ed223a3/0xd5401cbabc020c2e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x715:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f14ec expref: 502 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.992324:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x715:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.992328:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x715:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.992331:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520935.992332:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000bbdbe055/0xd5401cbabc020c0b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x710:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14a6 expref: 502 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.992334:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ad9880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993179648:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.992338:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025fa2 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x715:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.992385:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025f78 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x710:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.992406:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36165 00010000:00010000:0.0:1783520935.992408:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009ed223a3/0xd5401cbabc020c2e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x715:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14ec expref: 502 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.992421:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+502:2388:x1870153002432000:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.992421:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993180672 00010000:00010000:0.0:1783520935.992422:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009ed223a3/0xd5401cbabc020c2e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x715:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14ec expref: 502 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.992427:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993180672 00010000:00010000:3.0:1783520935.992437:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.992445:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000bbdbe055/0xd5401cbabc020c0b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x710:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14a6 expref: 502 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.992446:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351dc40380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993180800:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.992462:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44581 00000100:00100000:2.0:1783520935.992474:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+507:3628:x1870152993180672:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.992476:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000bbdbe055/0xd5401cbabc020c0b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x710:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14a6 expref: 502 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.992484:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c016a80 going to sleep for 11 seconds 00002000:00080000:2.0:1783520935.992506:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1814 count 1 00010000:00010000:2.0:1783520935.992520:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025fa9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x716:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.992530:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009e70076b/0xd5401cbabc022ace lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x716:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f31be expref: 502 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.992537:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x716:0x0].0x0 since=36s v=(0 0) 00010000:00010000:2.0:1783520935.992540:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x716:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520935.992543:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025fa9 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x716:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.992561:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009e70076b/0xd5401cbabc022ace lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x716:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31be expref: 502 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.992570:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009e70076b/0xd5401cbabc022ace lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x716:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31be expref: 502 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.992581:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ad9880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993180928:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.992622:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff9436298a6480 going to sleep for 11 seconds 00010000:00010000:3.0:1783520935.992628:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000bbdbe055/0xd5401cbabc020c0b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x710:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14a6 expref: 502 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.992640:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x710:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.992644:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x710:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.992650:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025f78 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x710:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.992658:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025f78 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x710:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.992680:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.992693:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025f78 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x710:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.992700:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025f78 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x710:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.992710:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025f78 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x710:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.992714:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025f78 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x710:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.992726:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.992741:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.992747:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025f78 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x710:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.992754:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.992758:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025f78 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x710:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.992763:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025f78 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x710:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.992773:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025f78 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x710:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.992778:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc025f78 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x710:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.992783:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+501:2388:x1870153002432000:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 365us (632us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.992793:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36165 00000100:00100000:2.0:1783520935.992797:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94360550f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+508:3626:x1870152993179136:12345-0@lo:6:osp-syn-1-0.0 Request processed in 1910us (2247us total) trans 4294976076 rc 0/0 00010000:00010000:0.0:1783520935.992798:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.992802:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.992805:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520935.992806:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44574 00000004:00080000:3.0:1783520935.992893:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cba9180/2, rc 0, transno 4294976076 00000100:00100000:3.0:1783520935.993068:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993179136:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.993195:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575ea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993179904:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.993212:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025f86 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x711:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.993215:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11f800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993179776:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.993238:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f7f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x70f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.993248:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1783520935.993251:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575df80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993180160:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.993259:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350384b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993181056:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.993270:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025f94 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x712:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.993277:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11d180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993180032:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520935.993286:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.993295:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025f8d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x713:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.993327:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993181056 00000100:00100000:2.0:1783520935.993333:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993181056 00000100:00100000:2.0:1783520935.993398:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44582 00000100:00100000:2.0:1783520935.993407:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503849180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+508:3626:x1870152993181056:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520935.993419:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1815 count 1 00000100:00100000:0.0:1783520935.993429:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511adaa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993180544:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.993430:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025fb0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x717:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.993445:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ab81d0e2/0xd5401cbabc020c3c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x717:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1508 expref: 500 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.993450:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025f9b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x714:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.993451:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x717:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520935.993456:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x717:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.993465:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025fb0 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x717:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.993507:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ab81d0e2/0xd5401cbabc020c3c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x717:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1508 expref: 500 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.993518:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ab81d0e2/0xd5401cbabc020c3c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x717:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1508 expref: 500 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.993529:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511adbb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993181184:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520935.993559:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351dc40380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993180800:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.993574:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967e40 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.993578:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025fa2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x715:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.994029:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.994069:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002432128 00000100:00100000:2.0:1783520935.994074:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002432128 00010000:00010000:2.0:1783520935.994102:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004cb354b0/0xd5401cbabc020c12 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x711:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14b4 expref: 501 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.994126:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36166 00000100:00100000:2.0:1783520935.994132:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575df80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+501:2388:x1870153002432128:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.994138:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.994142:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004cb354b0/0xd5401cbabc020c12 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x711:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14b4 expref: 501 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.994149:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004cb354b0/0xd5401cbabc020c12 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x711:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14b4 expref: 501 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.994160:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004cb354b0/0xd5401cbabc020c12 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x711:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14b4 expref: 501 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.994173:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x711:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.994181:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x711:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.994186:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025f86 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x711:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.994194:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.994204:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025f86 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x711:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.994230:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.994230:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002432384 00000100:00100000:1.0:1783520935.994236:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002432384 00010000:00010000:3.0:1783520935.994243:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025f86 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x711:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.994253:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025f86 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x711:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.994259:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000303d326a/0xd5401cbabc020c20 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x713:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14d0 expref: 501 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.994266:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575df80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+501:2388:x1870153002432128:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 130us (233us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.994267:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025f86 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x711:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.994275:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.994289:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36166 00010000:00010000:3.0:1783520935.994295:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025f86 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x711:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.994311:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002432640 00000100:00100000:2.0:1783520935.994316:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002432640 00010000:00010000:3.0:1783520935.994319:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.994324:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36167 00010000:00010000:2.0:1783520935.994333:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000007d16a6a/0xd5401cbabc020c04 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x70f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1498 expref: 501 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.994335:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520935.994339:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351dc40380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+501:2388:x1870153002432384:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.994344:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025f86 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x711:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.994355:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.994392:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000303d326a/0xd5401cbabc020c20 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x713:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14d0 expref: 501 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.994400:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000303d326a/0xd5401cbabc020c20 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x713:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14d0 expref: 501 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.994416:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36168 00010000:00010000:1.0:1783520935.994416:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000303d326a/0xd5401cbabc020c20 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x713:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14d0 expref: 501 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.994422:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.994430:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025f86 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x711:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.994432:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x713:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520935.994433:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f1850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+500:2388:x1870153002432640:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520935.994440:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x713:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.994449:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025f8d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x713:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.994450:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.994453:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025f86 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x711:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.994459:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000007d16a6a/0xd5401cbabc020c04 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x70f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1498 expref: 500 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.994467:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025f8d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x713:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.994480:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+509:3628:x1870152993179264:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3434us (3560us total) trans 4294976077 rc 0/0 00010000:00010000:0.0:1783520935.994487:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025f86 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x711:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.994504:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000007d16a6a/0xd5401cbabc020c04 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x70f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1498 expref: 500 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.994507:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44576 00010000:00010000:1.0:1783520935.994511:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.994527:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc025f86 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x711:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.994538:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000007d16a6a/0xd5401cbabc020c04 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x70f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1498 expref: 500 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.994539:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351dc40380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+500:2388:x1870153002432384:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 204us (347us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.994553:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.994554:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36167 00010000:00010000:0.0:1783520935.994558:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.994562:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x70f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.994563:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.994568:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x70f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.994574:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f7f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.994578:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025f8d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x713:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.994586:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f7f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x70f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.994591:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025f8d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x713:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520935.994592:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cba8700/2, rc 0, transno 4294976077 00000100:00100000:1.0:1783520935.994610:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cba8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993179264:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.994617:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520935.994649:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f1850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+498:2388:x1870153002432640:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 220us (380us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.994658:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36168 00000100:00100000:3.0:1783520935.994668:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503848700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993181312:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.994702:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.994727:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993181312 00000100:00100000:1.0:1783520935.994732:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993181312 00000100:00100000:1.0:1783520935.994756:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44583 00000100:00100000:1.0:1783520935.994765:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436342b8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+509:3626:x1870152993181312:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520935.994779:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1816 count 1 00010000:00010000:1.0:1783520935.994791:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025fb7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x718:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.994803:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e2acb2a6/0xd5401cbabc020c43 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x718:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1516 expref: 497 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.994807:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x718:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.994810:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x718:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.994812:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025fb7 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x718:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.994825:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e2acb2a6/0xd5401cbabc020c43 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x718:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1516 expref: 497 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.994831:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e2acb2a6/0xd5401cbabc020c43 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x718:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1516 expref: 497 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.994838:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575ea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993181440:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.994869:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e240 going to sleep for 11 seconds 00010000:00010000:0.0:1783520935.994935:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025f8d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x713:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.994944:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025f8d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x713:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.994968:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.994988:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.995004:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025f8d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x713:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.995016:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.995021:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025f8d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x713:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.995032:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025f8d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x713:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.995051:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025f8d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x713:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.995059:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc025f8d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x713:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.995066:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+510:3625:x1870152993179520:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3802us (4100us total) trans 4294976078 rc 0/0 00000100:00100000:0.0:1783520935.995077:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44577 00010000:00010000:2.0:1783520935.995082:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.995086:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.995090:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520935.995101:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363d321f80/2, rc 0, transno 4294976078 00010000:00010000:0.0:1783520935.995107:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f7f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.995117:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f7f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.995119:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993179520:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.995134:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f7f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x70f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.995142:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f7f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x70f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.995161:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520935.995175:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9436342b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993181568:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.995176:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.995184:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f7f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x70f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.995195:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.995199:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f7f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x70f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.995208:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f7f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x70f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.995212:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.995226:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f7f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x70f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.995235:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc025f7f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x70f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.995253:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993181568 00000100:00100000:3.0:1783520935.995257:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993181568 00010000:00010000:2.0:1783520935.995263:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.995268:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94360550c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+511:3626:x1870152993179008:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4239us (4658us total) trans 4294976079 rc 0/0 00010000:00010000:2.0:1783520935.995269:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.995274:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520935.995280:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44575 00000100:00100000:3.0:1783520935.995287:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44584 00000100:00100000:3.0:1783520935.995294:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+511:3627:x1870152993181568:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520935.995305:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1817 count 1 00010000:00010000:3.0:1783520935.995316:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025fbe lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x719:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.995320:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943508059f80/2, rc 0, transno 4294976079 00010000:00010000:3.0:1783520935.995326:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000075b5cf96/0xd5401cbabc020c4a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x719:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1524 expref: 497 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.995336:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508059f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993179008:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.995338:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x719:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.995344:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x719:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.995348:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025fbe lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x719:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.995389:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000075b5cf96/0xd5401cbabc020c4a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x719:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1524 expref: 497 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.995398:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000075b5cf96/0xd5401cbabc020c4a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x719:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1524 expref: 497 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.995407:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9436342b8000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993181824:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.995421:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ada680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993181696:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.995458:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520935.995466:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8240 going to sleep for 11 seconds 00000100:00100000:3.0:1783520935.995485:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993181696 00000100:00100000:3.0:1783520935.995488:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993181696 00000100:00100000:3.0:1783520935.995522:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44585 00000100:00100000:3.0:1783520935.995527:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+511:3628:x1870152993181696:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520935.995535:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1818 count 1 00010000:00010000:3.0:1783520935.995543:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025fc5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x71a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.995552:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000017b5db4e/0xd5401cbabc020c51 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x71a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1532 expref: 497 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.995561:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x71a:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.995563:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x71a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.995565:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025fc5 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x71a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.995769:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000017b5db4e/0xd5401cbabc020c51 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x71a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1532 expref: 497 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.995778:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000017b5db4e/0xd5401cbabc020c51 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x71a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1532 expref: 497 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.995787:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351635b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993181952:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.995795:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.995822:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.995856:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002432768 00000100:00100000:3.0:1783520935.995862:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8780 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.995862:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002432768 00000100:00100000:3.0:1783520935.995881:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002432256 00000100:00100000:3.0:1783520935.995885:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002432256 00010000:00010000:1.0:1783520935.995887:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009ed223a3/0xd5401cbabc020c2e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x715:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14ec expref: 498 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.995902:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000137d5946/0xd5401cbabc020c19 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x712:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14c2 expref: 499 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.995918:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36169 00000100:00100000:1.0:1783520935.995928:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511adb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+499:2389:x1870153002432768:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.995937:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520935.995938:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36170 00010000:00010000:1.0:1783520935.995939:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.995947:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009ed223a3/0xd5401cbabc020c2e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x715:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14ec expref: 499 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.995952:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad9180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+499:2388:x1870153002432256:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520935.995955:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002432512 00000100:00100000:3.0:1783520935.995960:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002432512 00010000:00010000:2.0:1783520935.995966:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.995970:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009ed223a3/0xd5401cbabc020c2e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x715:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14ec expref: 500 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.995975:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000137d5946/0xd5401cbabc020c19 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x712:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14c2 expref: 500 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.995978:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000055b02f97/0xd5401cbabc020c27 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x714:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14de expref: 500 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.995989:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.995998:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009ed223a3/0xd5401cbabc020c2e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x715:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14ec expref: 500 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.996013:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520935.996019:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000137d5946/0xd5401cbabc020c19 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x712:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14c2 expref: 499 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.996023:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x715:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.996024:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002432896 00000100:00100000:3.0:1783520935.996029:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002432896 00010000:00010000:1.0:1783520935.996032:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x715:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.996033:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002433024 00010000:00010000:1.0:1783520935.996039:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025fa2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x715:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.996039:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002433024 00010000:00010000:3.0:1783520935.996044:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009e70076b/0xd5401cbabc022ace lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x716:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31be expref: 501 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.996045:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000137d5946/0xd5401cbabc020c19 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x712:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14c2 expref: 501 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.996056:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ab81d0e2/0xd5401cbabc020c3c lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x717:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1508 expref: 501 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.996059:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025fa2 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x715:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.996077:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36171 00010000:00010000:2.0:1783520935.996079:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x712:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.996086:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x712:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520935.996089:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351dc42300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+500:2388:x1870153002432512:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.996094:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025f94 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x712:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.996095:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36172 00010000:00010000:3.0:1783520935.996098:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520935.996100:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.996106:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000055b02f97/0xd5401cbabc020c27 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x714:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14de expref: 500 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.996108:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351dc40700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+500:2387:x1870153002432896:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.996119:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025f94 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x712:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.996122:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.996125:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000055b02f97/0xd5401cbabc020c27 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x714:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14de expref: 500 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.996131:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511adb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+500:2389:x1870153002432768:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 201us (340us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.996131:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009e70076b/0xd5401cbabc022ace lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x716:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31be expref: 500 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.996153:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36169 00010000:00010000:3.0:1783520935.996158:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000055b02f97/0xd5401cbabc020c27 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x714:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f14de expref: 500 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.996163:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009e70076b/0xd5401cbabc022ace lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x716:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31be expref: 500 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.996171:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x714:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.996174:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.996176:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x714:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520935.996180:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36173 00010000:00010000:0.0:1783520935.996180:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x716:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.996186:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x716:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.996189:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025fa9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x716:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.996193:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351dc40e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+498:2387:x1870153002433024:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.996199:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad9180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+498:2388:x1870153002432256:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 250us (373us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.996203:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025fa9 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x716:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.996209:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520935.996216:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36170 00010000:00010000:1.0:1783520935.996221:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ab81d0e2/0xd5401cbabc020c3c lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x717:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1508 expref: 498 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.996224:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520935.996236:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025f9b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x714:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.996237:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ab81d0e2/0xd5401cbabc020c3c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x717:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1508 expref: 497 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.996246:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351dc40700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+497:2387:x1870153002432896:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 140us (254us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520935.996256:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025f9b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x714:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.996266:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36172 00010000:00010000:1.0:1783520935.996270:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x717:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.996274:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x717:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.996280:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.996280:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025fb0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x717:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.996291:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025fb0 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x717:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.996292:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025f9b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x714:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.996303:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351dc42300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+496:2388:x1870153002432512:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 214us (367us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.996311:0:8410:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009e70076b/0xd5401cbabc022ace lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x716:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31be expref: 496 pid: 8408 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.996317:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025f9b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x714:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.996332:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36171 00010000:00010000:2.0:1783520935.996344:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025f9b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x714:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.996344:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520935.996349:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ad9880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993180928:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.996401:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351dc40e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+494:2387:x1870153002433024:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 210us (390us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520935.996410:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36173 00010000:00010000:2.0:1783520935.996428:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025f9b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x714:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.996433:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025fa2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x715:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.996443:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025fa9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x716:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.996444:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025f94 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x712:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.996445:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025fa2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x715:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.996454:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.996471:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025f94 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x712:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.996473:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025fa2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x715:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.996484:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.996506:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025f9b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x714:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.996513:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025f94 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x712:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.996514:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025fa2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x715:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.996516:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.996524:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025f9b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x714:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.996539:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025f94 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x712:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.996544:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025f9b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x714:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.996546:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520935.996563:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+512:3625:x1870152993180288:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4438us (4620us total) trans 4294976080 rc 0/0 00010000:00010000:0.0:1783520935.996565:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025f9b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x714:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.996576:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44579 00000004:00080000:1.0:1783520935.996576:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e11c380/2, rc 0, transno 4294976080 00010000:00010000:0.0:1783520935.996581:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc025f9b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x714:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.996591:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993180288:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.996650:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993182080:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.996654:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5bb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+512:3627:x1870152993180416:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4385us (4483us total) trans 4294976081 rc 0/0 00010000:00010000:0.0:1783520935.996660:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:3.0:1783520935.996665:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44580 00010000:00010000:0.0:1783520935.996667:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025fa2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x715:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.996676:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.996681:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025fa2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x715:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.996692:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025fa2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x715:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.996694:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.996708:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025fa2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x715:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.996718:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc025fa2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x715:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.996724:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993182080 00000100:00100000:3.0:1783520935.996729:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993182080 00010000:00010000:0.0:1783520935.996737:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.996741:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.996745:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.996756:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.996763:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44586 00010000:00010000:1.0:1783520935.996772:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520935.996774:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+512:3625:x1870152993182080:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.996780:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.996786:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025f94 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x712:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520935.996786:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cbab100/2, rc 0, transno 4294976081 00002000:00080000:3.0:1783520935.996792:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1819 count 1 00000100:00100000:0.0:1783520935.996806:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cbab100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993180416:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.996807:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.996810:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025fcc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x71b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.996814:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025f94 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x712:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.996825:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025fa9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x716:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.996838:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006cb1e101/0xd5401cbabc020c58 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x71b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1540 expref: 493 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.996843:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993182208:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.996843:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025fa9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x716:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.996844:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025f94 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x712:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.996860:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x71b:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.996867:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025fa9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x716:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.996874:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025fa9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x716:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.996875:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025f94 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x712:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.996883:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.996896:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc025f94 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x712:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.996901:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.996902:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x71b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.996910:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025fcc lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x71b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.996922:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520935.996927:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025fa9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x716:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.996931:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006cb1e101/0xd5401cbabc020c58 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x71b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1540 expref: 493 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.996947:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520935.996951:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006cb1e101/0xd5401cbabc020c58 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x71b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1540 expref: 493 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.996953:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025fa9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x716:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.996971:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516358700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993182336:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.996973:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025fa9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x716:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.996989:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025fa9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x716:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.996989:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635787450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+514:3627:x1870152993179392:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5391us (6024us total) trans 4294976082 rc 0/0 00010000:00010000:2.0:1783520935.997006:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc025fa9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x716:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.997006:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44578 00000100:00100000:3.0:1783520935.997011:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8a80 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.997012:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+514:3628:x1870152993180672:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4538us (4680us total) trans 4294976083 rc 0/0 00000100:00100000:0.0:1783520935.997026:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44581 00010000:00010000:2.0:1783520935.997038:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.997042:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.997046:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.997055:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520935.997058:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363d321c00/2, rc 0, transno 4294976082 00000100:00100000:0.0:1783520935.997067:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d321c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993179392:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.997079:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993182208 00000100:00100000:2.0:1783520935.997084:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993182208 00010000:00010000:0.0:1783520935.997096:0:35276:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ab81d0e2/0xd5401cbabc020c3c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x717:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1508 expref: 493 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.997105:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511adbb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993181184:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520935.997124:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44587 00010000:00010000:0.0:1783520935.997124:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025fb0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x717:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.997133:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025fb0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x717:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.997140:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+513:3626:x1870152993182208:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.997141:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025fb0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x717:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.997153:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025fb0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x717:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.997157:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025fb0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x717:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520935.997159:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1820 count 1 00000004:00080000:1.0:1783520935.997169:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c5bad80/2, rc 0, transno 4294976083 00010000:00010000:0.0:1783520935.997175:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.997176:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025fd3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x71c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.997176:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993180672:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.997187:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.997193:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025fb0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x717:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.997197:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008071491b/0xd5401cbabc020c5f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x71c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f154e expref: 492 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.997199:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.997204:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025fb0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x717:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.997214:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x71c:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.997219:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025fb0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x717:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.997220:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x71c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.997228:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025fd3 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x71c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.997237:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993182592:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.997241:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025fb0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x717:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.997252:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc025fb0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x717:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.997260:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008071491b/0xd5401cbabc020c5f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x71c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f154e expref: 492 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.997269:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.997273:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520935.997274:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520935.997278:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.997288:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008071491b/0xd5401cbabc020c5f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x71c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f154e expref: 492 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.997295:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993182464:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.997303:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ad8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993182720:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.997321:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993182592 00000100:00100000:1.0:1783520935.997326:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993182592 00000100:00100000:2.0:1783520935.997340:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967e40 going to sleep for 11 seconds 00000100:00100000:0.0:1783520935.997340:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520935.997382:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993182464 00000100:00100000:1.0:1783520935.997387:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993182464 00000100:00100000:0.0:1783520935.997402:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44588 00000100:00100000:0.0:1783520935.997411:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+515:3628:x1870152993182592:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520935.997422:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1822 count 1 00000100:00100000:1.0:1783520935.997428:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44589 00010000:00010000:0.0:1783520935.997436:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025fda lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x71e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.997442:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351dc41500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+515:3627:x1870152993182464:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520935.997450:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000062e73baf/0xd5401cbabc0229bd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x71e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f30ad expref: 492 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.997456:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x71e:0x0].0x0 since=36s v=(0 0) 00002000:00080000:1.0:1783520935.997459:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1821 count 1 00010000:00010000:0.0:1783520935.997460:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x71e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.997466:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025fda lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x71e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.997474:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575ea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993181440:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.997476:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025fe1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x71d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.997485:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000062e73baf/0xd5401cbabc0229bd lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x71e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30ad expref: 492 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.997509:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025fb7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x718:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.997512:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000049ffdcb3/0xd5401cbabc020c66 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x71d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f155c expref: 492 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.997514:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000062e73baf/0xd5401cbabc0229bd lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x71e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30ad expref: 492 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.997539:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x71d:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.997545:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x71d:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520935.997545:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351dc42a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993182848:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.997554:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025fe1 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x71d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.997581:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351635b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993181952:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.997582:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000049ffdcb3/0xd5401cbabc020c66 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x71d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f155c expref: 492 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.997592:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520935.997605:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000049ffdcb3/0xd5401cbabc020c66 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x71d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f155c expref: 492 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.997608:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025fc5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x71a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.997608:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c016cc0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.997641:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.997643:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575c000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993182976:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.997679:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3eb40 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.997685:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.997717:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002433152 00000100:00100000:0.0:1783520935.997717:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503849180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+516:3626:x1870152993181056:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4309us (4427us total) trans 4294976084 rc 0/0 00000100:00100000:3.0:1783520935.997723:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002433152 00000004:00080000:1.0:1783520935.997727:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350384b480/2, rc 0, transno 4294976084 00000100:00100000:0.0:1783520935.997730:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44582 00000100:00100000:1.0:1783520935.997742:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350384b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993181056:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.997748:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e2acb2a6/0xd5401cbabc020c43 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x718:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1516 expref: 493 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.997757:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002433280 00000100:00100000:2.0:1783520935.997763:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002433280 00010000:00010000:2.0:1783520935.997784:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000075b5cf96/0xd5401cbabc020c4a lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x719:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1524 expref: 494 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.997804:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1783520935.997810:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002433408 00000100:00100000:0.0:1783520935.997814:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993183104:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520935.997815:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002433408 00010000:00010000:2.0:1783520935.997832:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000017b5db4e/0xd5401cbabc020c51 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x71a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1532 expref: 495 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.997842:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520935.997868:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36174 00000100:00100000:2.0:1783520935.997877:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+495:2387:x1870153002433152:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520935.997886:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520935.997889:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36175 00010000:00010000:2.0:1783520935.997894:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e2acb2a6/0xd5401cbabc020c43 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x718:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1516 expref: 495 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.997902:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f2c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+495:2390:x1870153002433280:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.997915:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.997922:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e2acb2a6/0xd5401cbabc020c43 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x718:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1516 expref: 495 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.997938:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e2acb2a6/0xd5401cbabc020c43 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x718:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1516 expref: 495 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.997947:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x718:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.997951:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x718:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.997956:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025fb7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x718:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.997962:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000075b5cf96/0xd5401cbabc020c4a lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x719:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1524 expref: 495 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.997966:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025fb7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x718:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.997982:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000075b5cf96/0xd5401cbabc020c4a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x719:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1524 expref: 494 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.997997:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520935.998003:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36176 00000100:00100000:1.0:1783520935.998014:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+494:2389:x1870153002433408:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520935.998018:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025fb7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x718:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.998024:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520935.998027:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+494:2387:x1870153002433152:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 149us (437us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.998033:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025fb7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x718:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.998035:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000017b5db4e/0xd5401cbabc020c51 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x71a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1532 expref: 494 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.998045:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36174 00010000:00010000:3.0:1783520935.998056:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x719:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.998056:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025fb7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x718:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.998062:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000017b5db4e/0xd5401cbabc020c51 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x71a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1532 expref: 493 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.998066:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x719:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.998078:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025fbe lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x719:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.998080:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025fb7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x718:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.998093:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000017b5db4e/0xd5401cbabc020c51 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x71a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1532 expref: 493 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.998099:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000075b5cf96/0xd5401cbabc020c4a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x719:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1524 expref: 493 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.998105:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025fbe lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x719:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.998112:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520935.998121:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x71a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.998127:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:2.0:1783520935.998134:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9436342b8000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993181824:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520935.998134:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x71a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.998135:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025fb7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x718:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.998138:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.998145:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025fc5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.998150:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbaa00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.998158:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025fb7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x718:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.998159:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025fbe lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x719:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.998160:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025fc5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x71a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.998168:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f2c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+491:2390:x1870153002433280:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 266us (526us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.998189:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025fb7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x718:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.998194:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36175 00010000:00010000:2.0:1783520935.998212:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025fbe lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x719:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.998220:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025fb7 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x718:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.998221:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025fbe lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x719:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.998223:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.998244:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025fbe lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x719:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.998248:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc025fb7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x718:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.998251:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993183104 00000100:00100000:1.0:1783520935.998251:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+490:2389:x1870153002433408:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 237us (566us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520935.998258:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993183104 00010000:00010000:2.0:1783520935.998270:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025fbe lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x719:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.998271:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36176 00010000:00010000:0.0:1783520935.998285:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520935.998290:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025fbe lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x719:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.998291:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44590 00010000:00010000:2.0:1783520935.998291:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to regular blp list, 0 blwis in pool 00010000:00010000:0.0:1783520935.998303:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5200 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:3.0:1783520935.998304:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351dc41180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+516:3626:x1870152993183104:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.998305:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.998309:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025fbe lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x719:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.998318:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025fbe lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x719:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520935.998322:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1823 count 1 00010000:00010000:0.0:1783520935.998327:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025fbe lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x719:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.998333:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc025fbe lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x719:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.998337:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025fe8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x71f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.998345:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.998348:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.998350:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.998352:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000040651e98/0xd5401cbabc020c74 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x71f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1578 expref: 489 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.998386:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x71f:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.998390:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x71f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.998395:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025fe8 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x71f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.998418:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000040651e98/0xd5401cbabc020c74 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x71f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1578 expref: 489 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.998429:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000040651e98/0xd5401cbabc020c74 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x71f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1578 expref: 489 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.998441:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351635b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993183232:192.168.204.33@tcp:104: 00000004:00080000:0.0:1783520935.998474:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9436342b9500/2, rc 0, transno 4294976085 00000100:00100000:2.0:1783520935.998475:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+518:3627:x1870152993181568:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3180us (3264us total) trans 4294976085 rc 0/0 00000100:00100000:2.0:1783520935.998487:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44584 00000100:00100000:0.0:1783520935.998488:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9436342b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993181568:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.998504:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d83c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520935.998517:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436342b8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+517:3626:x1870152993181312:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3751us (3811us total) trans 4294976086 rc 0/0 00000100:00100000:1.0:1783520935.998527:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44583 00010000:00010000:3.0:1783520935.998530:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025fc5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.998542:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025fc5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.998551:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025fc5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.998559:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025fc5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x71a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.998564:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351dc43480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993183360:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.998585:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.998594:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520935.998600:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520935.998602:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025fc5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x71a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.998612:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520935.998617:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025fc5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x71a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.998625:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993183360 00000100:00100000:1.0:1783520935.998631:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993183360 00000004:00080000:0.0:1783520935.998634:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503848700/2, rc 0, transno 4294976086 00010000:00010000:2.0:1783520935.998635:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025fc5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x71a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.998649:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503848700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993181312:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.998652:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025fc5 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x71a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.998662:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44591 00010000:00010000:2.0:1783520935.998664:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc025fc5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x71a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.998673:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351dc42680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+518:3626:x1870152993183360:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520935.998685:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.998690:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.998694:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.998701:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+518:3628:x1870152993181696:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3172us (3241us total) trans 4294976087 rc 0/0 00000004:00080000:1.0:1783520935.998708:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943511ada680/2, rc 0, transno 4294976087 00000100:00100000:0.0:1783520935.998709:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351dc42d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993183488:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520935.998714:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44585 00000100:00100000:1.0:1783520935.998724:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ada680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993181696:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.998744:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00080000:1.0:1783520935.998745:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1824 count 1 00010000:00010000:1.0:1783520935.998759:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025fef lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x720:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.998771:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007ffb6dc7/0xd5401cbabc020c7b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x720:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1586 expref: 489 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.998778:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x720:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520935.998782:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x720:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.998786:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025fef lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x720:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.998811:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351dc43b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993183616:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.998824:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993183488 00000100:00100000:0.0:1783520935.998829:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993183488 00000100:00100000:1.0:1783520935.998846:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.998866:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993183616 00000100:00100000:0.0:1783520935.998869:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993183616 00000100:00100000:3.0:1783520935.998876:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44592 00000100:00100000:3.0:1783520935.998885:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351dc41f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+519:3627:x1870152993183488:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520935.998896:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44593 00002000:00080000:3.0:1783520935.998897:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1825 count 1 00000100:00100000:0.0:1783520935.998906:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+519:3628:x1870152993183616:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520935.998910:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025ff6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x721:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520935.998919:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1826 count 1 00010000:00010000:3.0:1783520935.998920:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000dba0a16c/0xd5401cbabc020c90 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x721:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1594 expref: 489 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.998926:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007ffb6dc7/0xd5401cbabc020c7b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x720:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1586 expref: 489 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.998931:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x721:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520935.998935:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ffd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x722:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.998939:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x721:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.998948:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025ff6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x721:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.998951:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007ffb6dc7/0xd5401cbabc020c7b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x720:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1586 expref: 489 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.998953:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520935.998961:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000433f1ab9/0xd5401cbabc020c82 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x722:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f15a2 expref: 489 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.998984:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x722:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520935.998985:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000dba0a16c/0xd5401cbabc020c90 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x721:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1594 expref: 489 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.998985:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520935.998985:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575df80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993183744:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.998995:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x722:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520935.999005:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ffd lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x722:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.999007:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000dba0a16c/0xd5401cbabc020c90 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x721:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1594 expref: 489 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.999026:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.999031:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520935.999032:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351dc73800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993183872:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520935.999048:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002433536 00010000:00010000:0.0:1783520935.999048:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000433f1ab9/0xd5401cbabc020c82 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x722:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15a2 expref: 489 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.999054:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002433536 00000100:00100000:2.0:1783520935.999057:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002433792 00000100:00100000:3.0:1783520935.999060:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8240 going to sleep for 11 seconds 00000100:00100000:2.0:1783520935.999063:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002433792 00010000:00010000:0.0:1783520935.999071:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000433f1ab9/0xd5401cbabc020c82 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x722:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15a2 expref: 491 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.999077:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006cb1e101/0xd5401cbabc020c58 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x71b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1540 expref: 491 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.999086:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516358700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993182336:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520935.999088:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008071491b/0xd5401cbabc020c5f lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x71c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f154e expref: 491 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520935.999102:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351dc40700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993184000:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.999105:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025fcc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x71b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.999119:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002433664 00000100:00100000:1.0:1783520935.999124:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002433664 00000100:00100000:3.0:1783520935.999134:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36177 00000100:00100000:0.0:1783520935.999134:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c016240 going to sleep for 11 seconds 00010000:00010000:1.0:1783520935.999143:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000062e73baf/0xd5401cbabc0229bd lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x71e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30ad expref: 492 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.999145:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36178 00000100:00100000:3.0:1783520935.999149:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511adad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+492:2388:x1870153002433536:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520935.999161:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511adb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+492:2388:x1870153002433792:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520935.999162:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351dc42a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993182848:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520935.999163:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520935.999171:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006cb1e101/0xd5401cbabc020c58 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x71b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1540 expref: 492 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.999179:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520935.999183:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025fda lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x71e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.999186:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006cb1e101/0xd5401cbabc020c58 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x71b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1540 expref: 492 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.999187:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36179 00010000:00010000:2.0:1783520935.999188:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008071491b/0xd5401cbabc020c5f lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x71c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f154e expref: 492 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.999204:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+492:2388:x1870153002433664:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520935.999220:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006cb1e101/0xd5401cbabc020c58 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x71b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1540 expref: 492 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.999220:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520935.999227:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008071491b/0xd5401cbabc020c5f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x71c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f154e expref: 492 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.999232:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000062e73baf/0xd5401cbabc0229bd lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x71e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30ad expref: 492 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.999234:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x71b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.999242:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x71b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520935.999250:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025fcc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.999254:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x71c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520935.999257:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000062e73baf/0xd5401cbabc0229bd lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x71e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30ad expref: 491 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.999260:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025fcc lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x71b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.999262:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x71c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520935.999273:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025fd3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.999288:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025fd3 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x71c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.999293:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000062e73baf/0xd5401cbabc0229bd lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x71e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30ad expref: 491 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.999294:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520935.999310:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520935.999311:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x71e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520935.999311:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025fcc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520935.999316:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511adad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+490:2388:x1870153002433536:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 170us (369us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520935.999322:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x71e:0x0].0x0 since=0s v=(1 0) 00000100:00100000:3.0:1783520935.999333:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36177 00010000:00010000:1.0:1783520935.999333:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025fda lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.999336:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025fcc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.999337:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511adb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+490:2388:x1870153002433792:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 179us (355us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520935.999383:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36178 00010000:00010000:1.0:1783520935.999384:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025fda lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x71e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.999399:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025fcc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.999408:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025fcc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x71b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.999412:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520935.999426:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520935.999434:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008071491b/0xd5401cbabc020c5f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x71c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f154e expref: 488 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.999438:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520935.999441:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+488:2388:x1870153002433664:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 240us (413us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520935.999446:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025fcc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x71b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520935.999459:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36179 00000100:00100000:2.0:1783520935.999463:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ad8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993182720:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520935.999469:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.999475:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025fcc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x71b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.999482:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025fd3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.999483:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025fcc lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x71b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.999511:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025fcc lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x71b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.999513:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025fd3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.999532:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc025fcc lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x71b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.999547:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025fd3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.999557:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025fd3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.999557:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.999561:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520935.999564:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520935.999570:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025fd3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x71c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.999576:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025fda lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.999585:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025fda lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520935.999587:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520935.999598:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025fda lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.999600:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520935.999608:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025fd3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x71c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.999632:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520935.999633:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025fda lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x71e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.999637:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025fd3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x71c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.999656:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025fd3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x71c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520935.999663:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520935.999669:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025fd3 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x71c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.999674:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520935.999682:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025fda lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x71e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.999683:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc025fd3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x71c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.999699:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+520:3626:x1870152993182208:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2560us (2816us total) trans 4294976088 rc 0/0 00000004:00080000:0.0:1783520935.999702:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351575f100/2, rc 0, transno 4294976088 00010000:00010000:1.0:1783520935.999705:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520935.999712:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025fda lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x71e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520935.999718:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44587 00000100:00100000:0.0:1783520935.999719:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993182208:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.999739:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025fda lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x71e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520935.999742:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.999746:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520935.999750:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.999755:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025fda lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x71e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.999765:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc025fda lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x71e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520935.999787:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520935.999790:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.999794:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351dc42a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993184128:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520935.999796:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520935.999834:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520935.999877:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+520:3628:x1870152993182592:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2467us (2606us total) trans 4294976089 rc 0/0 00000100:00100000:3.0:1783520935.999878:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993184128 00000100:00100000:3.0:1783520935.999885:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993184128 00000100:00100000:0.0:1783520935.999889:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44588 00000004:00080000:1.0:1783520935.999895:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351575ce00/2, rc 0, transno 4294976089 00000100:00100000:1.0:1783520935.999913:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993182592:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.999951:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351dc42300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993184256:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520935.999979:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.000033:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993184256 00000100:00100000:0.0:1783520936.000037:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+521:3625:x1870152993182080:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3266us (3345us total) trans 4294976090 rc 0/0 00000100:00100000:2.0:1783520936.000039:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993184256 00000100:00100000:0.0:1783520936.000045:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44586 00000004:00080000:1.0:1783520936.000056:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351575d880/2, rc 0, transno 4294976090 00000100:00100000:0.0:1783520936.000066:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44594 00000100:00100000:1.0:1783520936.000076:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993182080:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.000077:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44595 00000100:00100000:2.0:1783520936.000093:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+521:3626:x1870152993184256:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.000109:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1828 count 1 00000100:00100000:0.0:1783520936.000110:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351dc40e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993184384:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.000127:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026004 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x724:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.000135:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.000146:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e8628e39/0xd5401cbabc020c97 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x724:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f15be expref: 486 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.000152:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x724:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.000156:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x724:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.000158:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026004 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x724:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.000162:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993184384 00000100:00100000:3.0:1783520936.000167:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993184384 00000100:00100000:0.0:1783520936.000174:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516358700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+522:3625:x1870152993184128:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.000177:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e8628e39/0xd5401cbabc020c97 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x724:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15be expref: 486 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.000190:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e8628e39/0xd5401cbabc020c97 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x724:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15be expref: 486 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.000192:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1827 count 1 00000100:00100000:3.0:1783520936.000201:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44596 00010000:00010000:0.0:1783520936.000212:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02600b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x723:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.000213:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ad8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993184512:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.000217:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351dc40380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+522:3627:x1870152993184384:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520936.000236:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1829 count 1 00010000:00010000:0.0:1783520936.000236:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004746b48c/0xd5401cbabc020c89 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x723:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f15b0 expref: 486 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.000246:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x723:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.000250:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026012 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x725:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.000250:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x723:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.000261:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02600b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x723:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.000266:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967a80 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.000270:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000804033f8/0xd5401cbabc020c9e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x725:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f15cc expref: 486 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.000281:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x725:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.000285:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x725:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.000289:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026012 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x725:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.000291:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004746b48c/0xd5401cbabc020c89 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x723:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15b0 expref: 486 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.000315:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004746b48c/0xd5401cbabc020c89 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x723:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15b0 expref: 486 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.000328:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11d180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993184640:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.000332:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000804033f8/0xd5401cbabc020c9e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x725:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15cc expref: 486 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.000335:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575c000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993182976:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.000388:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025fe1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x71d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.000430:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.000431:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c016a80 going to sleep for 10 seconds 00010000:00010000:3.0:1783520936.000471:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000804033f8/0xd5401cbabc020c9e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x725:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15cc expref: 486 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.000478:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002433920 00000100:00100000:3.0:1783520936.000482:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e006d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993184768:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.000485:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002433920 00010000:00010000:1.0:1783520936.000522:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000049ffdcb3/0xd5401cbabc020c66 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x71d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f155c expref: 487 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.000530:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8a80 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.000554:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36180 00000100:00100000:1.0:1783520936.000564:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575c000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+487:2390:x1870153002433920:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.000573:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.000579:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000049ffdcb3/0xd5401cbabc020c66 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x71d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f155c expref: 487 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.000590:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000049ffdcb3/0xd5401cbabc020c66 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x71d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f155c expref: 487 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.000605:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000049ffdcb3/0xd5401cbabc020c66 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x71d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f155c expref: 487 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.000616:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x71d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.000619:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x71d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.000624:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025fe1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.000633:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025fe1 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x71d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.000652:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.000661:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025fe1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.000671:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025fe1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.000679:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575c000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+486:2390:x1870153002433920:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 116us (249us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.000683:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025fe1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.000691:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36180 00010000:00010000:2.0:1783520936.000698:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025fe1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x71d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.000718:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.000728:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.000732:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025fe1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x71d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.000738:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbbe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.000741:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025fe1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x71d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.000747:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025fe1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x71d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.000756:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025fe1 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x71d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.000764:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc025fe1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x71d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.000776:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.000777:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.000778:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.000808:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351dc41500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+523:3627:x1870152993182464:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3370us (3465us total) trans 4294976091 rc 0/0 00000100:00100000:2.0:1783520936.000816:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44589 00000004:00080000:0.0:1783520936.000816:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351575fb80/2, rc 0, transno 4294976091 00000100:00100000:0.0:1783520936.000828:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993182464:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.000874:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1783520936.000881:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993184896:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.000902:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.001044:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993184896 00000100:00100000:3.0:1783520936.001049:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993184896 00000100:00100000:3.0:1783520936.001076:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44597 00000100:00100000:3.0:1783520936.001085:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+523:3627:x1870152993184896:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520936.001135:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1830 count 1 00010000:00010000:3.0:1783520936.001146:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026019 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x726:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.001154:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f1dc4dbf/0xd5401cbabc0228ac lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x726:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f9c expref: 485 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.001159:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x726:0x0].0x0 since=36s v=(0 0) 00010000:00010000:3.0:1783520936.001161:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x726:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.001163:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026019 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x726:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.001177:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f1dc4dbf/0xd5401cbabc0228ac lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x726:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f9c expref: 485 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.001186:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f1dc4dbf/0xd5401cbabc0228ac lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x726:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f9c expref: 485 pid: 8410 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.001192:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993185024:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.001217:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943502ee5cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.003512:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575df80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993183744:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.003527:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351dc40700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993184000:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.003532:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351dc73800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993183872:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.003538:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.003545:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025fef lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x720:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.003554:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ffd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x722:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.003555:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025ff6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x721:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.003579:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.003591:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351635b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993183232:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.003611:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.003611:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025fe8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x71f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.003626:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002434048 00000100:00100000:3.0:1783520936.003632:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002434048 00000100:00100000:2.0:1783520936.003640:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002434432 00000100:00100000:2.0:1783520936.003646:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002434432 00010000:00010000:3.0:1783520936.003665:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007ffb6dc7/0xd5401cbabc020c7b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x720:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1586 expref: 487 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.003669:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000040651e98/0xd5401cbabc020c74 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x71f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1578 expref: 487 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.003709:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002434304 00000100:00100000:3.0:1783520936.003713:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002434304 00000100:00100000:2.0:1783520936.003726:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36181 00010000:00010000:3.0:1783520936.003730:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000433f1ab9/0xd5401cbabc020c82 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x722:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15a2 expref: 488 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.003742:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad8a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+488:2387:x1870153002434048:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.003756:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.003759:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36182 00010000:00010000:2.0:1783520936.003766:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007ffb6dc7/0xd5401cbabc020c7b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x720:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1586 expref: 488 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.003771:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36183 00000100:00100000:3.0:1783520936.003772:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357f050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+488:2389:x1870153002434432:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.003782:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511adb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+488:2387:x1870153002434304:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.003785:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.003792:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000040651e98/0xd5401cbabc020c74 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x71f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1578 expref: 488 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.003795:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007ffb6dc7/0xd5401cbabc020c7b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x720:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1586 expref: 488 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.003821:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000040651e98/0xd5401cbabc020c74 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x71f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1578 expref: 488 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.003834:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007ffb6dc7/0xd5401cbabc020c7b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x720:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1586 expref: 488 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.003841:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000040651e98/0xd5401cbabc020c74 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x71f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1578 expref: 488 pid: 35276 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.003853:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x720:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.003859:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x71f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.003859:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.003860:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x720:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.003868:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x71f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.003868:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000433f1ab9/0xd5401cbabc020c82 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x722:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15a2 expref: 486 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.003872:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025fef lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x720:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.003875:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025fe8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.003885:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000433f1ab9/0xd5401cbabc020c82 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x722:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15a2 expref: 486 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.003897:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025fe8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x71f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.003903:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000433f1ab9/0xd5401cbabc020c82 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x722:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15a2 expref: 486 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.003924:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025fef lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x720:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.003927:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x722:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.003934:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.003935:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x722:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.003944:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ffd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x722:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.003954:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ffd lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x722:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.003960:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025fe8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.003963:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.003973:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.003974:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025fe8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.003975:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357f050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+485:2389:x1870153002434432:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 207us (396us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.003990:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad8a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+485:2387:x1870153002434048:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 252us (457us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.003996:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36182 00010000:00010000:1.0:1783520936.003996:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025fe8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x71f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.004000:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511adb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+485:2387:x1870153002434304:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 218us (388us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.004008:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36181 00000100:00100000:0.0:1783520936.004016:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36183 00010000:00010000:1.0:1783520936.004026:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025fe8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x71f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.004049:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ffd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x722:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.004052:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520936.004053:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.004062:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ffd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x722:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.004070:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002434176 00010000:00010000:0.0:1783520936.004072:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ffd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x722:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.004074:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002434176 00010000:00010000:0.0:1783520936.004084:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ffd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x722:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.004096:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000dba0a16c/0xd5401cbabc020c90 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x721:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1594 expref: 483 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.004101:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.004113:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025fe8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x71f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.004126:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.004131:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025fe8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x71f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.004134:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36184 00010000:00010000:0.0:1783520936.004148:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025fe8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x71f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.004150:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c3b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+483:2387:x1870153002434176:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.004165:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.004168:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025fe8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x71f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.004173:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000dba0a16c/0xd5401cbabc020c90 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x721:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1594 expref: 483 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.004192:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc025fe8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x71f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.004198:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000dba0a16c/0xd5401cbabc020c90 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x721:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1594 expref: 483 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.004217:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.004221:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.004223:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.004235:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000dba0a16c/0xd5401cbabc020c90 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x721:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1594 expref: 483 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.004245:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351dc41180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+524:3626:x1870152993183104:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5940us (6400us total) trans 4294976092 rc 0/0 00000100:00100000:1.0:1783520936.004263:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44590 00010000:00010000:2.0:1783520936.004265:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x721:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.004270:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x721:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.004274:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025ff6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x721:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.004283:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025ff6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x721:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.004302:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351575ea00/2, rc 0, transno 4294976092 00000100:00100000:1.0:1783520936.004315:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993183104:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.004406:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993185152:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.004439:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.004487:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993185152 00000100:00100000:1.0:1783520936.004503:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993185152 00000100:00100000:1.0:1783520936.004528:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44598 00000100:00100000:1.0:1783520936.004535:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+524:3625:x1870152993185152:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.004558:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1831 count 1 00010000:00010000:1.0:1783520936.004593:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026020 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x727:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.004609:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008c84636e/0xd5401cbabc020cd6 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x727:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f15e8 expref: 482 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.004617:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x727:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.004620:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x727:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.004625:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026020 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x727:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.004647:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008c84636e/0xd5401cbabc020cd6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x727:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15e8 expref: 482 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.004658:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008c84636e/0xd5401cbabc020cd6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x727:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15e8 expref: 482 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.004671:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435080d3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993185280:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.004727:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3eb40 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.004741:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.004743:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.004756:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.004761:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ffd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x722:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.004770:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c3b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+482:2387:x1870153002434176:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 621us (716us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.004777:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb200 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520936.004783:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36184 00010000:00010000:3.0:1783520936.004784:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ffd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x722:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.004795:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ffd lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x722:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.004809:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ffd lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x722:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.004817:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc025ffd lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x722:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.004833:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.004833:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ad8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993184512:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.004838:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.004842:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.004858:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026004 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x724:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.004859:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025ff6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x721:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.004874:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+525:3628:x1870152993183616:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5970us (6028us total) trans 4294976093 rc 0/0 00010000:00010000:3.0:1783520936.004882:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025ff6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x721:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.004885:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44593 00010000:00010000:3.0:1783520936.004892:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025ff6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x721:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.004899:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025ff6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x721:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.004910:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025fef lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x720:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.004912:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.004922:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11d180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993184640:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.004927:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025fef lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x720:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.004938:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025fef lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x720:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.004943:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02600b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x723:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.004952:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.004962:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025ff6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x721:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.004968:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025fef lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x720:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.004980:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.004987:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025ff6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x721:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.004992:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.004997:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025ff6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x721:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.005008:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351dc41f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+525:3627:x1870152993183488:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6123us (6262us total) trans 4294976094 rc 0/0 00010000:00010000:1.0:1783520936.005013:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025ff6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x721:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.005018:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44592 00010000:00010000:1.0:1783520936.005028:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc025ff6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x721:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.005051:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.005056:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025fef lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x720:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.005061:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e006d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993184768:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.005080:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.005083:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026012 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x725:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.005084:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351dc42680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+525:3626:x1870152993183360:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6413us (6483us total) trans 4294976095 rc 0/0 00010000:00010000:1.0:1783520936.005087:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025fef lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x720:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.005101:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44591 00010000:00010000:1.0:1783520936.005112:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025fef lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x720:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.005131:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351dc43480/2, rc 0, transno 4294976095 00010000:00010000:1.0:1783520936.005132:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025fef lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x720:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.005142:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc025fef lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x720:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.005146:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351dc43480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993183360:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.005169:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.005170:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.005175:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.005178:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.005212:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.005226:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1783520936.005239:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.005239:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435080d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993185408:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.005278:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.005291:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000004:00080000:1.0:1783520936.005334:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351dc43b80/2, rc 0, transno 4294976093 00000100:00100000:3.0:1783520936.005339:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993185024:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.005348:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351dc43b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993183616:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.005385:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026019 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x726:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.005395:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002434560 00000100:00100000:2.0:1783520936.005401:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002434560 00000100:00100000:1.0:1783520936.005423:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:2.0:1783520936.005424:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e8628e39/0xd5401cbabc020c97 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x724:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15be expref: 482 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.005432:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435055cdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993185536:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.005453:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002434816 00000100:00100000:2.0:1783520936.005458:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002434816 00000100:00100000:1.0:1783520936.005462:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.005475:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000804033f8/0xd5401cbabc020c9e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x725:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15cc expref: 483 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.005508:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002434688 00000100:00100000:2.0:1783520936.005511:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002434688 00010000:00010000:2.0:1783520936.005523:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004746b48c/0xd5401cbabc020c89 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x723:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15b0 expref: 484 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.005539:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520936.005563:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351dc42d80/2, rc 0, transno 4294976094 00000100:00100000:0.0:1783520936.005573:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351dc42d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993183488:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.005619:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1783520936.005627:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993185664:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.005640:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002434944 00000100:00100000:3.0:1783520936.005644:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002434944 00000100:00100000:1.0:1783520936.005647:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36185 00000100:00100000:2.0:1783520936.005658:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36186 00010000:00010000:3.0:1783520936.005663:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f1dc4dbf/0xd5401cbabc0228ac lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x726:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f9c expref: 485 pid: 8410 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.005663:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.005666:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+485:2388:x1870153002434560:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.005676:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e006d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+485:2388:x1870153002434816:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.005684:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.005688:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.005692:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e8628e39/0xd5401cbabc020c97 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x724:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15be expref: 485 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.005694:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36187 00010000:00010000:2.0:1783520936.005698:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000804033f8/0xd5401cbabc020c9e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x725:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15cc expref: 485 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.005709:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36188 00000100:00100000:3.0:1783520936.005710:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351dc73800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+485:2388:x1870153002434688:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.005719:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e8628e39/0xd5401cbabc020c97 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x724:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15be expref: 485 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.005723:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350ade1880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+485:2389:x1870153002434944:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.005724:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.005730:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000804033f8/0xd5401cbabc020c9e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x725:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15cc expref: 485 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.005734:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004746b48c/0xd5401cbabc020c89 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x723:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15b0 expref: 485 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.005737:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.005744:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f1dc4dbf/0xd5401cbabc0228ac lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x726:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f9c expref: 485 pid: 8410 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.005748:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e8628e39/0xd5401cbabc020c97 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x724:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15be expref: 485 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.005766:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000804033f8/0xd5401cbabc020c9e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x725:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15cc expref: 485 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.005769:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f1dc4dbf/0xd5401cbabc0228ac lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x726:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f9c expref: 485 pid: 8410 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.005781:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004746b48c/0xd5401cbabc020c89 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x723:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15b0 expref: 484 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.005782:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x724:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.005792:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x724:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.005797:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x725:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.005802:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026004 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x724:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.005803:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f1dc4dbf/0xd5401cbabc0228ac lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x726:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f9c expref: 483 pid: 8410 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.005804:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x725:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.005807:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004746b48c/0xd5401cbabc020c89 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x723:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15b0 expref: 483 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.005817:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026012 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x725:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.005826:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026004 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x724:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.005831:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x726:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.005833:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x723:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.005840:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x726:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.005841:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x723:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.005847:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026012 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x725:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.005851:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02600b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x723:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.005851:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026019 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x726:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.005863:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.005873:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026019 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x726:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.005874:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02600b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x723:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.005889:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.005893:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+481:2388:x1870153002434560:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 235us (726us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.005897:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.005898:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.005902:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36185 00000100:00100000:2.0:1783520936.005920:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e006d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+480:2388:x1870153002434816:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 249us (708us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.005923:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993185408 00000100:00100000:1.0:1783520936.005929:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993185408 00000100:00100000:0.0:1783520936.005930:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350ade1880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+480:2389:x1870153002434944:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 207us (637us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.005931:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351dc73800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+480:2388:x1870153002434688:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 227us (694us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.005937:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36186 00000100:00100000:0.0:1783520936.005946:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36188 00000100:00100000:1.0:1783520936.005952:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993185536 00000100:00100000:1.0:1783520936.005956:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993185536 00010000:00010000:2.0:1783520936.005980:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026004 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x724:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.005983:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993185664 00000100:00100000:1.0:1783520936.005988:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993185664 00000100:00100000:3.0:1783520936.005992:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36187 00000100:00100000:0.0:1783520936.005992:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44599 00010000:00010000:2.0:1783520936.005996:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026004 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x724:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.006004:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435080d0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+527:3625:x1870152993185408:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.006010:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026004 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x724:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.006020:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026004 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x724:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.006021:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026012 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x725:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.006023:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1832 count 1 00000100:00100000:1.0:1783520936.006026:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44600 00010000:00010000:0.0:1783520936.006040:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026027 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x728:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.006042:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c9a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+527:3628:x1870152993185536:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.006049:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026012 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x725:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.006058:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026012 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x725:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.006061:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.006062:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ec867fcd/0xd5401cbabc020cac lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x728:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f15f6 expref: 477 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.006063:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1833 count 1 00010000:00010000:3.0:1783520936.006072:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026012 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x725:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.006075:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x728:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.006078:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x728:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.006082:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02602e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x729:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.006083:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026027 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x728:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.006088:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520936.006099:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520936.006105:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026004 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x724:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.006106:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.006113:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026012 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x725:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.006122:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.006127:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026004 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x724:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.006134:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.006138:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026012 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x725:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.006139:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026004 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x724:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.006142:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000058df32d1/0xd5401cbabc020cb3 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x729:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1604 expref: 477 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.006153:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026012 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x725:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.006158:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026004 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x724:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.006158:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x729:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.006169:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x729:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.006172:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026004 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x724:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.006172:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026012 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x725:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.006176:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02602e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x729:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.006186:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026012 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x725:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.006189:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+528:3626:x1870152993184256:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6099us (6206us total) trans 4294976096 rc 0/0 00010000:00010000:3.0:1783520936.006195:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.006197:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.006201:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.006208:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44595 00010000:00010000:0.0:1783520936.006213:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.006217:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.006220:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.006222:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000058df32d1/0xd5401cbabc020cb3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x729:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1604 expref: 477 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.006236:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000058df32d1/0xd5401cbabc020cb3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x729:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1604 expref: 477 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.006243:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ec867fcd/0xd5401cbabc020cac lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x728:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15f6 expref: 477 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.006244:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351dc42300/2, rc 0, transno 4294976096 00000100:00100000:1.0:1783520936.006255:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502c9b800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993185792:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.006263:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ec867fcd/0xd5401cbabc020cac lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x728:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15f6 expref: 477 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.006265:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351dc42300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993184256:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.006284:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351dc40700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993185920:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.006292:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44601 00000100:00100000:3.0:1783520936.006301:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351dc40380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+528:3627:x1870152993184384:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6088us (6164us total) trans 4294976097 rc 0/0 00000100:00100000:2.0:1783520936.006308:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351635b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+528:3627:x1870152993185664:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.006311:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.006315:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44596 00002000:00080000:2.0:1783520936.006329:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1834 count 1 00000100:00100000:0.0:1783520936.006333:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435038ec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993186048:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.006345:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026019 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x726:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.006347:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026035 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x72a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.006382:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.006385:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026019 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x726:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.006395:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026019 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x726:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.006406:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001f1022b1/0xd5401cbabc020cba lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x72a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1612 expref: 477 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.006407:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026019 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x726:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.006428:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351dc40e00/2, rc 0, transno 4294976097 00010000:00010000:2.0:1783520936.006431:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x72a:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.006436:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x72a:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.006438:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351dc40e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993184384:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.006442:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026035 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x72a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.006443:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.006449:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.006459:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026019 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x726:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.006470:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.006476:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026019 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x726:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.006480:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1783520936.006490:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993186048 00010000:00010000:1.0:1783520936.006505:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026019 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x726:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.006506:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993186048 00000100:00100000:0.0:1783520936.006507:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435038ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993186176:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.006528:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026019 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x726:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.006537:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026019 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x726:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.006538:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44602 00000100:00100000:0.0:1783520936.006543:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.006547:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+529:3627:x1870152993184896:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5463us (5642us total) trans 4294976098 rc 0/0 00000100:00100000:2.0:1783520936.006554:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435038efb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+529:3625:x1870152993186048:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.006566:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44597 00002000:00080000:2.0:1783520936.006572:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1835 count 1 00010000:00010000:1.0:1783520936.006572:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.006577:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520936.006580:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351575c700/2, rc 0, transno 4294976098 00010000:00010000:1.0:1783520936.006581:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.006583:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993186176 00000100:00100000:3.0:1783520936.006587:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993186176 00010000:00010000:2.0:1783520936.006589:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02603c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x72b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.006593:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993184896:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.006610:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a2d0bcaf/0xd5401cbabc020cc1 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x72b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1620 expref: 477 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.006611:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44603 00000100:00100000:3.0:1783520936.006622:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435038eea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+529:3625:x1870152993186176:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.006624:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x72b:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.006632:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x72b:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.006633:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435038eca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993186304:0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520936.006638:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1836 count 1 00010000:00010000:2.0:1783520936.006639:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02603c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x72b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.006654:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026043 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x72c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.006664:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.006669:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a2d0bcaf/0xd5401cbabc020cc1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x72b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1620 expref: 477 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.006670:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b1f24350/0xd5401cbabc020ccf lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x72c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f162e expref: 477 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.006692:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x72c:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.006692:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a2d0bcaf/0xd5401cbabc020cc1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x72b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1620 expref: 477 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.006700:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x72c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.006709:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026043 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x72c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.006710:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ad8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993186432:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.006735:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c016a80 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.006741:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b1f24350/0xd5401cbabc020ccf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x72c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f162e expref: 477 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.006748:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511a4c600 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.006748:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993186304 00000100:00100000:0.0:1783520936.006754:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993186304 00010000:00010000:2.0:1783520936.006782:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001f1022b1/0xd5401cbabc020cba lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x72a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1612 expref: 477 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.006783:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44604 00000100:00100000:0.0:1783520936.006796:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435038ece00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+530:3627:x1870152993186304:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.006810:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001f1022b1/0xd5401cbabc020cba lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x72a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1612 expref: 477 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.006813:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b1f24350/0xd5401cbabc020ccf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x72c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f162e expref: 477 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.006813:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1837 count 1 00010000:00010000:0.0:1783520936.006831:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02604a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x72d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.006838:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ad9c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993186688:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.006840:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993186560:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.006846:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b0dd0cf9/0xd5401cbabc020cc8 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x72d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f163c expref: 477 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.006853:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x72d:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.006857:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x72d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.006860:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02604a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x72d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.006878:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511a4c6c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.006884:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8240 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.006886:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b0dd0cf9/0xd5401cbabc020cc8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x72d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f163c expref: 477 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.006895:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b0dd0cf9/0xd5401cbabc020cc8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x72d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f163c expref: 477 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.006906:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435038ee680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993186816:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.006934:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c016240 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.006949:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02600b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x723:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.006959:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02600b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x723:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.006968:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02600b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x723:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.006975:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02600b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x723:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.007004:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.007017:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.007024:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02600b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x723:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.007032:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.007034:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02600b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x723:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.007041:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02600b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x723:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.007050:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02600b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x723:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.007056:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02600b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x723:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.007068:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.007071:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.007072:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.007089:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516358700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+531:3625:x1870152993184128:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7015us (7255us total) trans 4294976099 rc 0/0 00000100:00100000:0.0:1783520936.007096:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44594 00000004:00080000:1.0:1783520936.007122:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351dc42a00/2, rc 0, transno 4294976099 00000100:00100000:1.0:1783520936.007141:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351dc42a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993184128:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.007318:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1783520936.007329:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435038ef100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993186944:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.007378:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.007416:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993186944 00000100:00100000:2.0:1783520936.007422:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993186944 00000100:00100000:2.0:1783520936.007454:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44605 00000100:00100000:2.0:1783520936.007462:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c99500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+531:3627:x1870152993186944:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.007475:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1838 count 1 00010000:00010000:2.0:1783520936.007489:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026051 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x72e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.007516:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004d92f5b0/0xd5401cbabc022b06 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x72e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f31f6 expref: 477 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.007522:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x72e:0x0].0x0 since=36s v=(0 0) 00010000:00010000:2.0:1783520936.007525:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x72e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.007528:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026051 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x72e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.007550:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004d92f5b0/0xd5401cbabc022b06 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x72e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31f6 expref: 477 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.007559:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004d92f5b0/0xd5401cbabc022b06 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x72e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31f6 expref: 477 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.007568:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511adad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993187072:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.007627:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967a80 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.008051:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435080d3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993185280:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.008074:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026020 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x727:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.008075:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502c9b800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993185792:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.008101:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02602e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x729:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.008113:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351dc40700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993185920:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.008132:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026027 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x728:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.008568:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ad9c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993186688:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.008586:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ad8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993186432:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.008589:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026035 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x72a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.008644:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02603c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x72b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.008652:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993186560:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.008679:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026043 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x72c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.009580:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.009600:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435038ee680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993186816:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.009610:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511adad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993187072:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.009629:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.009639:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02604a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x72d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.009649:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026051 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x72e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.009682:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002435200 00000100:00100000:3.0:1783520936.009689:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002435200 00000100:00100000:1.0:1783520936.009698:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.009726:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008c84636e/0xd5401cbabc020cd6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x727:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15e8 expref: 478 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.009731:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.009732:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002435456 00000100:00100000:2.0:1783520936.009739:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002435456 00010000:00010000:2.0:1783520936.009761:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a2d0bcaf/0xd5401cbabc020cc1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x72b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1620 expref: 479 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.009771:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.009798:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.009809:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002435712 00000100:00100000:3.0:1783520936.009813:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002435712 00000100:00100000:2.0:1783520936.009816:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002435968 00000100:00100000:0.0:1783520936.009817:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002435072 00000100:00100000:1.0:1783520936.009822:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.009823:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002435968 00000100:00100000:0.0:1783520936.009825:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002435072 00010000:00010000:3.0:1783520936.009829:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b1f24350/0xd5401cbabc020ccf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x72c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f162e expref: 480 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.009843:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002435328 00000100:00100000:3.0:1783520936.009846:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002435328 00010000:00010000:2.0:1783520936.009850:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004d92f5b0/0xd5401cbabc022b06 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x72e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31f6 expref: 482 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.009853:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000058df32d1/0xd5401cbabc020cb3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x729:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1604 expref: 483 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.009861:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ec867fcd/0xd5401cbabc020cac lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x728:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15f6 expref: 483 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.009863:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.009888:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002435584 00000100:00100000:1.0:1783520936.009892:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002435840 00000100:00100000:2.0:1783520936.009894:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002435584 00000100:00100000:1.0:1783520936.009898:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002435840 00000100:00100000:0.0:1783520936.009905:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36189 00000100:00100000:3.0:1783520936.009913:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36190 00010000:00010000:2.0:1783520936.009918:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001f1022b1/0xd5401cbabc020cba lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x72a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1612 expref: 485 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.009919:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357f450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+485:2390:x1870153002435200:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.009924:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b0dd0cf9/0xd5401cbabc020cc8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x72d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f163c expref: 485 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.009927:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c99c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+485:2390:x1870153002435456:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.009939:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.009942:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.009951:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a2d0bcaf/0xd5401cbabc020cc1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x72b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1620 expref: 485 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.009951:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008c84636e/0xd5401cbabc020cd6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x727:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15e8 expref: 485 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.009958:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36191 00000100:00100000:1.0:1783520936.009970:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36192 00000100:00100000:2.0:1783520936.009973:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c9b800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+485:2390:x1870153002435712:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.009981:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a2d0bcaf/0xd5401cbabc020cc1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x72b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1620 expref: 485 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.009985:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.009985:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008c84636e/0xd5401cbabc020cd6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x727:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15e8 expref: 485 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.009988:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a80380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+485:2390:x1870153002435968:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.009994:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b1f24350/0xd5401cbabc020ccf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x72c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f162e expref: 485 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.010005:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.010016:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004d92f5b0/0xd5401cbabc022b06 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x72e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31f6 expref: 485 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.010023:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b1f24350/0xd5401cbabc020ccf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x72c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f162e expref: 485 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.010050:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a2d0bcaf/0xd5401cbabc020cc1 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x72b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1620 expref: 485 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.010052:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004d92f5b0/0xd5401cbabc022b06 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x72e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31f6 expref: 485 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.010052:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008c84636e/0xd5401cbabc020cd6 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x727:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15e8 expref: 485 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.010055:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b1f24350/0xd5401cbabc020ccf lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x72c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f162e expref: 485 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.010088:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x72c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.010088:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x727:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.010091:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x72b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.010095:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x72c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.010096:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004d92f5b0/0xd5401cbabc022b06 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x72e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31f6 expref: 482 pid: 178496 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.010098:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x727:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.010100:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x72b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.010106:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026043 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x72c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.010110:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026020 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x727:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.010114:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02603c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x72b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.010129:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x72e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.010132:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026043 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x72c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.010139:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x72e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.010140:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026020 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x727:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.010143:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02603c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x72b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.010149:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026051 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x72e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.010167:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.010172:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026051 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x72e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.010187:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.010191:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.010192:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.010223:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c9b800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+481:2390:x1870153002435712:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 250us (521us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.010226:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357f450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+481:2390:x1870153002435200:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 309us (646us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.010227:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c99c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+481:2390:x1870153002435456:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 300us (596us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.010234:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a80380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+481:2390:x1870153002435968:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 248us (502us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.010242:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36191 00000100:00100000:0.0:1783520936.010252:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36189 00000100:00100000:3.0:1783520936.010259:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36190 00000100:00100000:1.0:1783520936.010269:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36192 00000100:00100000:2.0:1783520936.010277:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36193 00000100:00100000:2.0:1783520936.010287:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a80e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+477:2389:x1870153002435072:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520936.010293:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36194 00010000:00010000:2.0:1783520936.010297:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.010301:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36195 00010000:00010000:2.0:1783520936.010304:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000058df32d1/0xd5401cbabc020cb3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x729:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1604 expref: 477 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.010308:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a83100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+477:2389:x1870153002435328:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.010316:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a83480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+477:2389:x1870153002435584:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.010316:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36196 00010000:00010000:3.0:1783520936.010327:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.010331:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.010334:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000058df32d1/0xd5401cbabc020cb3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x729:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1604 expref: 477 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.010335:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357d850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+477:2389:x1870153002435840:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.010336:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ec867fcd/0xd5401cbabc020cac lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x728:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15f6 expref: 477 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.010340:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001f1022b1/0xd5401cbabc020cba lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x72a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1612 expref: 477 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.010352:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.010388:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000058df32d1/0xd5401cbabc020cb3 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x729:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1604 expref: 477 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.010391:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b0dd0cf9/0xd5401cbabc020cc8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x72d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f163c expref: 477 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.010404:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x729:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.010409:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b0dd0cf9/0xd5401cbabc020cc8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x72d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f163c expref: 476 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.010412:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x729:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.010420:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001f1022b1/0xd5401cbabc020cba lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x72a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1612 expref: 476 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.010421:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02602e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x729:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.010425:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ec867fcd/0xd5401cbabc020cac lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x728:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15f6 expref: 476 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.010447:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02602e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x729:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.010461:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001f1022b1/0xd5401cbabc020cba lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x72a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1612 expref: 476 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.010466:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b0dd0cf9/0xd5401cbabc020cc8 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x72d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f163c expref: 476 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.010476:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.010488:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ec867fcd/0xd5401cbabc020cac lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x728:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f15f6 expref: 475 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.010502:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x72a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.010505:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x72d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.010507:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x72a:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.010513:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a80e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+474:2389:x1870153002435072:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 226us (741us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.010516:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x72d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.010519:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026035 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x72a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.010524:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x728:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.010527:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02604a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x72d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.010535:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x728:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.010535:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36193 00010000:00010000:1.0:1783520936.010545:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026035 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x72a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.010548:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026027 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x728:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.010549:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02604a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x72d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.010567:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02602e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x729:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.010575:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026027 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x728:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.010579:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.010586:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02602e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x729:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.010599:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02602e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x729:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.010605:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a83480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+472:2389:x1870153002435584:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 293us (785us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.010607:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.010622:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02602e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x729:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.010627:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36195 00000100:00100000:3.0:1783520936.010638:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a83100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+472:2389:x1870153002435328:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 334us (837us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.010653:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520936.010654:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36194 00010000:00010000:1.0:1783520936.010658:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026051 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x72e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.010665:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026051 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x72e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.010673:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026051 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x72e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.010680:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.010681:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026051 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x72e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.010688:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02602e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x729:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.010698:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.010703:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02602e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x729:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.010704:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.010716:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02602e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x729:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.010736:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02602e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x729:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.010743:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02602e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x729:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.010765:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.010769:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026051 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x72e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.010776:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.010780:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026051 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x72e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.010787:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026051 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x72e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.010798:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026051 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x72e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.010806:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026051 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x72e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.010821:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.010823:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.010826:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.010839:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026027 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x728:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.010848:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c99500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+533:3627:x1870152993186944:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3383us (3487us total) trans 4294976101 rc 0/0 00000100:00100000:2.0:1783520936.010851:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c9a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+533:3628:x1870152993185536:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4811us (5388us total) trans 4294976100 rc 0/0 00010000:00010000:3.0:1783520936.010855:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026027 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x728:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.010867:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44605 00010000:00010000:3.0:1783520936.010876:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026027 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x728:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.010884:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44600 00010000:00010000:3.0:1783520936.010886:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026027 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x728:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.010901:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.010911:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.010915:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026027 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x728:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.010918:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435038ef100/2, rc 0, transno 4294976101 00000004:00080000:2.0:1783520936.010922:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435055cdf80/2, rc 0, transno 4294976100 00010000:00010000:3.0:1783520936.010923:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.010929:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026027 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x728:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.010935:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026027 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x728:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.010938:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435055cdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993185536:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.010939:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435038ef100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993186944:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.010955:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026027 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x728:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.010965:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026027 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x728:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.010975:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026035 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x72a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.010982:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.010986:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.010989:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026035 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x72a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.010992:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.011007:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026035 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x72a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.011016:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026035 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x72a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.011041:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.011067:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.011073:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026035 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x72a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.011083:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.011087:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026035 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x72a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.011095:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026035 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x72a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.011108:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435080d0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+533:3625:x1870152993185408:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5106us (5828us total) trans 4294976102 rc 0/0 00010000:00010000:2.0:1783520936.011110:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026035 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x72a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.011116:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435080d1f80/2, rc 0, transno 4294976102 00000100:00100000:1.0:1783520936.011126:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435080d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993185408:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.011129:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026035 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x72a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.011153:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.011156:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.011157:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351635b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+533:3627:x1870152993185664:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4857us (5502us total) trans 4294976103 rc 0/0 00010000:00010000:2.0:1783520936.011159:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.011162:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44599 00000100:00100000:1.0:1783520936.011168:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44601 00000004:00080000:1.0:1783520936.011193:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e11d180/2, rc 0, transno 4294976103 00000100:00100000:1.0:1783520936.011196:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993185664:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.011203:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026043 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x72c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.011215:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026043 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x72c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.011224:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026043 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x72c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.011232:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026043 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x72c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.011246:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.011271:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.011278:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026043 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x72c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.011287:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.011292:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026043 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x72c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.011300:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026043 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x72c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.011310:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026043 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x72c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.011315:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026043 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x72c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.011316:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435038eea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+532:3625:x1870152993186176:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4696us (4778us total) trans 4294976104 rc 0/0 00000100:00100000:3.0:1783520936.011330:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44603 00000004:00080000:1.0:1783520936.011338:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435038ec000/2, rc 0, transno 4294976104 00000100:00100000:1.0:1783520936.011345:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435038ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993186176:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.011349:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.011352:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.011354:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.011473:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02604a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x72d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.011477:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.011486:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02604a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x72d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.011501:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02604a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x72d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.011502:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.011511:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02604a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x72d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.011528:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.011538:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.011543:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02604a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x72d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.011550:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.011552:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02604a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x72d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.011561:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02604a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x72d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.011573:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02604a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x72d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.011579:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435038ee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993187200:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.011581:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02604a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x72d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.011604:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.011607:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.011610:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.011617:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435038ece00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+532:3627:x1870152993186304:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4822us (4950us total) trans 4294976105 rc 0/0 00000100:00100000:2.0:1783520936.011626:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44604 00000100:00100000:1.0:1783520936.011635:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.011635:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435038ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993187328:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.011653:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993187200 00000100:00100000:2.0:1783520936.011660:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993187200 00000100:00100000:1.0:1783520936.011675:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435038edf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993187712:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.011701:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44606 00000100:00100000:1.0:1783520936.011707:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.011715:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635786850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+532:3625:x1870152993187200:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.011723:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993187712 00000100:00100000:3.0:1783520936.011729:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993187712 00002000:00080000:2.0:1783520936.011736:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1839 count 1 00010000:00010000:2.0:1783520936.011757:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026058 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x72f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.011763:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44607 00000100:00100000:3.0:1783520936.011777:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+533:3625:x1870152993187712:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.011780:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000062a2dbda/0xd5401cbabc020cf2 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x72f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1658 expref: 470 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.011791:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x72f:0x0].0x0 since=37s v=(0 0) 00002000:00080000:3.0:1783520936.011794:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1843 count 1 00010000:00010000:2.0:1783520936.011796:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x72f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.011803:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026058 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x72f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.011812:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02605f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x733:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.011829:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.011833:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000062a2dbda/0xd5401cbabc020cf2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x72f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1658 expref: 470 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.011841:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000cb888eef/0xd5401cbabc020d00 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x733:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1690 expref: 470 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.011851:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x733:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.011853:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000062a2dbda/0xd5401cbabc020cf2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x72f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1658 expref: 470 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.011857:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x733:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.011867:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02605f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x733:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.011868:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435038ee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993187456:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.011879:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511adad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993187840:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.011905:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.011906:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000cb888eef/0xd5401cbabc020d00 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x733:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1690 expref: 470 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.011920:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000cb888eef/0xd5401cbabc020d00 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x733:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1690 expref: 470 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.011925:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993187328 00000100:00100000:1.0:1783520936.011930:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993187328 00000004:00080000:0.0:1783520936.011935:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435038eca80/2, rc 0, transno 4294976105 00000100:00100000:3.0:1783520936.011937:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993187968:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.011951:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435038eca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993186304:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.011958:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993187456 00000100:00100000:2.0:1783520936.011959:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967a80 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.011961:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1783520936.011962:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993187456 00000100:00100000:3.0:1783520936.011973:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8a80 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.011976:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435038eed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993187584:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.011985:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44608 00000100:00100000:1.0:1783520936.011993:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44609 00000100:00100000:2.0:1783520936.011998:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435038ec380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+535:3626:x1870152993187328:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.012004:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435038ed180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+535:3627:x1870152993187456:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.012004:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00080000:2.0:1783520936.012013:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1840 count 1 00002000:00080000:1.0:1783520936.012020:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1841 count 1 00010000:00010000:2.0:1783520936.012028:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026066 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x730:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.012030:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993187584 00000100:00100000:3.0:1783520936.012037:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993187584 00010000:00010000:1.0:1783520936.012037:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02606d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x731:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.012046:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b643aae0/0xd5401cbabc02137b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x730:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1666 expref: 470 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.012058:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x730:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.012058:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006dfe9afb/0xd5401cbabc020cf9 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x731:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1674 expref: 470 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.012067:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x730:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.012070:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44610 00010000:00010000:2.0:1783520936.012077:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026066 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x730:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.012080:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x731:0x0].0x0 since=37s v=(0 0) 00000100:00100000:0.0:1783520936.012082:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351dc73800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993188096:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.012086:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e006d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+536:3627:x1870152993187584:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.012090:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x731:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.012100:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02606d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x731:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.012106:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1842 count 1 00000100:00100000:0.0:1783520936.012114:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.012125:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026074 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x732:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.012133:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b643aae0/0xd5401cbabc02137b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x730:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1666 expref: 470 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.012138:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006dfe9afb/0xd5401cbabc020cf9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x731:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1674 expref: 470 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.012145:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002c5b13be/0xd5401cbabc020d3f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x732:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1682 expref: 470 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.012153:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b643aae0/0xd5401cbabc02137b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x730:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1666 expref: 470 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.012168:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006dfe9afb/0xd5401cbabc020cf9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x731:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1674 expref: 470 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.012169:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357d850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+470:2389:x1870153002435840:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 1838us (2309us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.012171:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x732:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.012181:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x732:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.012182:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511adb480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993188224:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.012187:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36196 00010000:00010000:3.0:1783520936.012192:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026074 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x732:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.012197:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993188352:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.012219:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57d80 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.012229:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002c5b13be/0xd5401cbabc020d3f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x732:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1682 expref: 469 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.012236:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.012241:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002c5b13be/0xd5401cbabc020d3f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x732:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1682 expref: 469 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.012242:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993188096 00000100:00100000:0.0:1783520936.012248:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993188096 00000100:00100000:3.0:1783520936.012254:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993188480:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.012279:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44611 00000100:00100000:3.0:1783520936.012285:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d83c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.012290:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435080d3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+537:3626:x1870152993188096:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.012303:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1844 count 1 00010000:00010000:0.0:1783520936.012316:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02607b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x734:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.012329:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000042e1fa76/0xd5401cbabc020d1c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x734:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f169e expref: 469 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.012337:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x734:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.012340:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x734:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.012345:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02607b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x734:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.012403:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026020 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x727:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.012403:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000042e1fa76/0xd5401cbabc020d1c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x734:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f169e expref: 469 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.012413:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026020 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x727:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.012416:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000042e1fa76/0xd5401cbabc020d1c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x734:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f169e expref: 469 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.012424:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026020 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x727:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.012427:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502c9b800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993188608:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.012432:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026020 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x727:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.012446:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.012456:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.012464:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026020 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x727:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.012469:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c016cc0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.012476:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.012480:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026020 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x727:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.012488:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02603c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x72b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.012490:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026020 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x727:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.012526:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02603c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x72b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.012534:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026020 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x727:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.012539:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02603c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x72b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.012546:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026020 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x727:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.012551:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+538:3625:x1870152993185152:12345-0@lo:6:osp-syn-1-0.0 Request processed in 8015us (8110us total) trans 4294976106 rc 0/0 00010000:00010000:0.0:1783520936.012555:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02603c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x72b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.012566:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44598 00010000:00010000:2.0:1783520936.012592:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.012595:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520936.012596:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351575df80/2, rc 0, transno 4294976106 00010000:00010000:2.0:1783520936.012598:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.012610:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993185152:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.012628:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.012640:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.012647:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02603c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x72b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.012648:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:2.0:1783520936.012658:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185a00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520936.012659:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993188736:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.012663:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02603c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x72b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.012671:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02603c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x72b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.012680:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02603c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x72b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.012685:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02603c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x72b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.012692:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.012705:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.012709:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.012711:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.012717:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993188736 00000100:00100000:1.0:1783520936.012722:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993188736 00000100:00100000:1.0:1783520936.012747:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44612 00000100:00100000:1.0:1783520936.012755:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c99c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+539:3628:x1870152993188736:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.012767:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1845 count 1 00010000:00010000:1.0:1783520936.012781:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026082 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x735:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.012789:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435038efb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+539:3625:x1870152993186048:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6240us (6407us total) trans 4294976107 rc 0/0 00010000:00010000:1.0:1783520936.012793:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000052895dc2/0xd5401cbabc020d07 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x735:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f16ac expref: 469 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.012797:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44602 00010000:00010000:1.0:1783520936.012801:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x735:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.012804:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x735:0x0].0x0 since=0s v=(0 0) 00000004:00080000:2.0:1783520936.012807:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435038ec700/2, rc 0, transno 4294976107 00010000:00010000:1.0:1783520936.012809:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026082 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x735:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.012824:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435038ec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993186048:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.012840:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000052895dc2/0xd5401cbabc020d07 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x735:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16ac expref: 469 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.012853:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000052895dc2/0xd5401cbabc020d07 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x735:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16ac expref: 469 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.012864:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff2d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993188864:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.012877:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993188992:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.012898:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.012901:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e3c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.012925:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993188992 00000100:00100000:3.0:1783520936.012930:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993188992 00000100:00100000:3.0:1783520936.012963:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44613 00000100:00100000:3.0:1783520936.012972:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a80380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+539:3628:x1870152993188992:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520936.012984:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1846 count 1 00010000:00010000:3.0:1783520936.013036:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026089 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x736:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.013049:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ac9f1a32/0xd5401cbabc0229f5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x736:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f30e5 expref: 469 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.013061:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x736:0x0].0x0 since=36s v=(0 0) 00010000:00010000:3.0:1783520936.013064:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x736:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.013068:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026089 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x736:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.013087:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ac9f1a32/0xd5401cbabc0229f5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x736:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30e5 expref: 469 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.013098:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ac9f1a32/0xd5401cbabc0229f5 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x736:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30e5 expref: 469 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.013110:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993189120:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.013151:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8240 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.015869:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993187968:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.015875:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993189120:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.015918:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026089 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x736:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.015918:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02605f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x733:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.015927:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511adb480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993188224:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.015955:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026066 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x730:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.015980:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993188352:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.015992:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502c9b800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993188608:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.015993:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.015998:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511adad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993187840:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.015999:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02606d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x731:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.016017:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02607b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x734:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.016027:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026058 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x72f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.016033:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.016035:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff2d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993188864:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.016055:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026082 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x735:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.016068:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002436608 00000100:00100000:3.0:1783520936.016069:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.016076:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002436608 00000100:00100000:1.0:1783520936.016088:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002436352 00000100:00100000:1.0:1783520936.016093:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002436352 00000100:00100000:2.0:1783520936.016100:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002436736 00000100:00100000:3.0:1783520936.016102:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.016108:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002436736 00010000:00010000:1.0:1783520936.016116:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ac9f1a32/0xd5401cbabc0229f5 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x736:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30e5 expref: 471 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.016116:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b643aae0/0xd5401cbabc02137b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x730:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1666 expref: 471 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.016135:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002c5b13be/0xd5401cbabc020d3f lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x732:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1682 expref: 472 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.016138:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.016162:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.016172:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002436992 00000100:00100000:1.0:1783520936.016177:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002436992 00000100:00100000:0.0:1783520936.016183:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002436096 00000100:00100000:2.0:1783520936.016184:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002436480 00000100:00100000:2.0:1783520936.016192:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002436480 00000100:00100000:0.0:1783520936.016192:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002436096 00010000:00010000:1.0:1783520936.016193:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000052895dc2/0xd5401cbabc020d07 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x735:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16ac expref: 473 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.016195:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.016213:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002436224 00010000:00010000:0.0:1783520936.016217:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000cb888eef/0xd5401cbabc020d00 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x733:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1690 expref: 475 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.016218:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002436224 00000100:00100000:3.0:1783520936.016222:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.016234:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006dfe9afb/0xd5401cbabc020cf9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x731:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1674 expref: 476 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.016240:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000062a2dbda/0xd5401cbabc020cf2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x72f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1658 expref: 476 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.016241:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002436864 00000100:00100000:3.0:1783520936.016246:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002436864 00000100:00100000:0.0:1783520936.016251:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36197 00010000:00010000:3.0:1783520936.016266:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000042e1fa76/0xd5401cbabc020d1c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x734:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f169e expref: 477 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.016269:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350ade3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+477:2390:x1870153002436352:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.016276:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36198 00000100:00100000:1.0:1783520936.016289:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36199 00010000:00010000:0.0:1783520936.016291:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.016294:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350ade1c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+477:2390:x1870153002436608:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.016302:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ac9f1a32/0xd5401cbabc0229f5 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x736:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30e5 expref: 477 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.016307:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36200 00000100:00100000:1.0:1783520936.016307:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350ade0700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+477:2390:x1870153002436736:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.016308:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.016321:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350ade1880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+477:2390:x1870153002436992:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.016321:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b643aae0/0xd5401cbabc02137b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x730:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1666 expref: 477 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.016324:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.016333:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ac9f1a32/0xd5401cbabc0229f5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x736:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30e5 expref: 477 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.016335:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.016336:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002c5b13be/0xd5401cbabc020d3f lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x732:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1682 expref: 477 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.016344:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000052895dc2/0xd5401cbabc020d07 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x735:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16ac expref: 477 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.016349:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b643aae0/0xd5401cbabc02137b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x730:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1666 expref: 477 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.016392:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002c5b13be/0xd5401cbabc020d3f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x732:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1682 expref: 477 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.016395:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000052895dc2/0xd5401cbabc020d07 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x735:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16ac expref: 477 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.016438:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b643aae0/0xd5401cbabc02137b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x730:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1666 expref: 477 pid: 35282 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.016438:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x732:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.016440:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000052895dc2/0xd5401cbabc020d07 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x735:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16ac expref: 477 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.016441:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ac9f1a32/0xd5401cbabc0229f5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x736:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30e5 expref: 477 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.016446:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x732:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.016458:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026074 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x732:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.016471:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x730:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.016473:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026074 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x732:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.016476:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x735:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.016478:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x730:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.016478:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x736:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.016487:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x735:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.016490:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026066 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x730:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.016490:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x736:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.016507:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026082 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x735:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.016512:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.016514:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026089 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x736:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.016529:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026066 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x730:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.016531:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026082 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x735:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.016547:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026089 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x736:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.016559:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350ade0700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+474:2390:x1870153002436736:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 256us (488us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.016566:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.016570:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36199 00010000:00010000:3.0:1783520936.016574:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.016597:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36201 00000100:00100000:2.0:1783520936.016599:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350ade1c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+473:2390:x1870153002436608:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 307us (608us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.016602:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.016609:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350ade1880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+473:2390:x1870153002436992:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 288us (504us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.016612:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351dc40700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+473:2387:x1870153002436096:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.016617:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36198 00000100:00100000:3.0:1783520936.016631:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36200 00010000:00010000:1.0:1783520936.016631:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.016633:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350ade3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+473:2390:x1870153002436352:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 366us (600us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.016642:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000cb888eef/0xd5401cbabc020d00 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x733:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1690 expref: 472 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.016644:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36202 00000100:00100000:0.0:1783520936.016655:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36197 00010000:00010000:1.0:1783520936.016659:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000cb888eef/0xd5401cbabc020d00 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x733:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1690 expref: 471 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.016661:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c3b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+471:2387:x1870153002436480:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520936.016667:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36203 00010000:00010000:2.0:1783520936.016680:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.016681:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36204 00000100:00100000:3.0:1783520936.016683:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357ec50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+470:2387:x1870153002436224:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.016684:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cb888eef/0xd5401cbabc020d00 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x733:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1690 expref: 470 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.016691:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006dfe9afb/0xd5401cbabc020cf9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x731:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1674 expref: 470 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.016700:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+470:2387:x1870153002436864:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.016701:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.016711:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x733:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.016713:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000062a2dbda/0xd5401cbabc020cf2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x72f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1658 expref: 469 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.016716:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x733:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.016719:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006dfe9afb/0xd5401cbabc020cf9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x731:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1674 expref: 469 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.016719:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.016728:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02605f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x733:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.016731:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000042e1fa76/0xd5401cbabc020d1c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x734:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f169e expref: 469 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.016746:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000062a2dbda/0xd5401cbabc020cf2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x72f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1658 expref: 469 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.016759:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02605f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x733:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.016769:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000042e1fa76/0xd5401cbabc020d1c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x734:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f169e expref: 469 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.016772:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006dfe9afb/0xd5401cbabc020cf9 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x731:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1674 expref: 469 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.016784:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000062a2dbda/0xd5401cbabc020cf2 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x72f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1658 expref: 469 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.016799:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x731:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.016808:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x731:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.016818:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02606d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x731:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.016819:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x72f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.016825:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.016829:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x72f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.016830:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000042e1fa76/0xd5401cbabc020d1c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x734:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f169e expref: 467 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.016839:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02606d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x731:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.016841:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026058 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x72f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.016851:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x734:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.016855:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351dc40700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+466:2387:x1870153002436096:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 244us (718us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.016861:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x734:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.016869:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026058 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x72f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.016872:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02607b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x734:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.016876:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36201 00010000:00010000:2.0:1783520936.016877:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.016891:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02607b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x734:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.016903:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.016905:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c3b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+465:2387:x1870153002436480:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 249us (740us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.016913:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.016915:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36202 00010000:00010000:1.0:1783520936.016916:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026089 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x736:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.016928:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026089 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x736:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.016933:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357ec50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+464:2387:x1870153002436224:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 251us (737us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.016939:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026089 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x736:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.016941:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+464:2387:x1870153002436864:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 245us (719us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.016943:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026066 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x730:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.016950:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36203 00010000:00010000:1.0:1783520936.016962:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026089 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x736:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.016965:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026066 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x730:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.016965:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36204 00010000:00010000:2.0:1783520936.016982:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026066 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x730:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.016987:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026066 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x730:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.016995:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.017009:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:0.0:1783520936.017012:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520936.017022:0:12734:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002c5b13be/0xd5401cbabc020d3f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x732:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1682 expref: 462 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.017022:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026089 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x736:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.017046:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.017053:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026089 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x736:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.017057:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993188480:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.017070:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026089 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x736:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.017077:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026074 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x732:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.017102:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026089 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x736:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.017103:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.017111:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026066 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x730:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.017114:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026089 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x736:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.017126:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.017131:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026066 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x730:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.017140:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026066 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x730:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.017143:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.017147:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.017152:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.017156:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435038ec380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+541:3626:x1870152993187328:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5160us (5328us total) trans 4294976109 rc 0/0 00000100:00100000:1.0:1783520936.017158:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a80380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+541:3628:x1870152993188992:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4187us (4257us total) trans 4294976108 rc 0/0 00010000:00010000:3.0:1783520936.017159:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026066 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x730:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.017174:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44608 00010000:00010000:3.0:1783520936.017179:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026066 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x730:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.017183:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44613 00000004:00080000:0.0:1783520936.017184:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435038ed500/2, rc 0, transno 4294976109 00000100:00100000:0.0:1783520936.017205:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435038ed500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993187328:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.017207:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.017212:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.017215:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520936.017224:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350ade3100/2, rc 0, transno 4294976108 00010000:00010000:3.0:1783520936.017226:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026074 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x732:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.017235:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993188992:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.017239:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026074 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x732:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.017250:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026074 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x732:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.017258:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026074 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x732:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.017267:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026082 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x735:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.017282:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.017284:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026082 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x735:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.017289:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.017299:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026074 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x732:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.017300:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a83480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993189376:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.017301:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026082 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x735:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.017324:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.017325:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026082 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x735:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.017330:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026074 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x732:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.017344:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.017347:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026074 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x732:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.017347:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.017394:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026074 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x732:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.017403:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026074 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x732:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.017407:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1783520936.017418:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e006d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+541:3627:x1870152993187584:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5335us (5411us total) trans 4294976110 rc 0/0 00000100:00100000:0.0:1783520936.017419:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a80e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993189248:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.017423:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.017431:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026082 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x735:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.017435:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44610 00010000:00010000:2.0:1783520936.017450:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.017454:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026082 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x735:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.017462:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.017469:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c99c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+540:3628:x1870152993188736:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4712us (4778us total) trans 4294976111 rc 0/0 00010000:00010000:2.0:1783520936.017471:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026082 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x735:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.017476:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435038eed80/2, rc 0, transno 4294976110 00000100:00100000:1.0:1783520936.017488:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44612 00000100:00100000:3.0:1783520936.017509:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435038eed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993187584:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.017509:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026082 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x735:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.017522:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026082 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x735:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.017532:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02605f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x733:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.017539:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.017543:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.017545:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02605f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x733:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.017546:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520936.017546:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514ff1500/2, rc 0, transno 4294976111 00010000:00010000:2.0:1783520936.017554:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.017558:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993188736:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.017561:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02605f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x733:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.017566:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993189376 00000100:00100000:1.0:1783520936.017570:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1783520936.017572:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993189376 00000100:00100000:1.0:1783520936.017582:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993189632:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.017584:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02605f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x733:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.017592:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993189248 00000100:00100000:3.0:1783520936.017598:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993189248 00010000:00010000:0.0:1783520936.017604:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.017610:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44614 00010000:00010000:0.0:1783520936.017615:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:2.0:1783520936.017623:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a83100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+541:3626:x1870152993189376:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.017623:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.017624:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02605f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x733:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.017638:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993189632 00010000:00010000:0.0:1783520936.017638:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5000 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:3.0:1783520936.017644:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993189632 00010000:00010000:0.0:1783520936.017645:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02605f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x733:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.017648:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1848 count 1 00000100:00100000:1.0:1783520936.017653:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993189504:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.017664:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02605f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x733:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.017667:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026090 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x738:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.017674:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44615 00010000:00010000:0.0:1783520936.017682:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02605f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x733:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.017683:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.017690:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c9b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+542:3626:x1870152993189248:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.017697:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000020646d0a/0xd5401cbabc020d23 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x738:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f16d6 expref: 461 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.017702:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc02605f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x733:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.017704:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993189504 00000100:00100000:1.0:1783520936.017709:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993189504 00002000:00080000:3.0:1783520936.017710:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1847 count 1 00010000:00010000:2.0:1783520936.017714:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x738:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.017723:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x738:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.017729:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026097 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x737:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.017730:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026090 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x738:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.017733:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44616 00010000:00010000:0.0:1783520936.017734:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.017739:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.017743:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.017745:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+543:3628:x1870152993189632:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.017770:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1850 count 1 00010000:00010000:3.0:1783520936.017772:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a95b3e11/0xd5401cbabc020d15 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x737:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f16c8 expref: 461 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.017778:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000020646d0a/0xd5401cbabc020d23 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x738:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16d6 expref: 461 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.017782:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc02609e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x73a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.017787:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x737:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.017797:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x737:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.017807:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026097 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x737:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.017812:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000020646d0a/0xd5401cbabc020d23 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x738:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16d6 expref: 461 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.017814:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006a0dc80a/0xd5401cbabc020d31 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x73a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f16f2 expref: 461 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.017831:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+544:3625:x1870152993187712:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6055us (6125us total) trans 4294976112 rc 0/0 00000100:00100000:2.0:1783520936.017842:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511adad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993189760:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.017845:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a95b3e11/0xd5401cbabc020d15 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x737:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16c8 expref: 461 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.017847:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x73a:0x0].0x0 since=37s v=(0 0) 00000100:00100000:0.0:1783520936.017851:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44607 00010000:00010000:1.0:1783520936.017856:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x73a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.017863:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc02609e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x73a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.017872:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a95b3e11/0xd5401cbabc020d15 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x737:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16c8 expref: 461 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.017886:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993189888:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.017889:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006a0dc80a/0xd5401cbabc020d31 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x73a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16f2 expref: 461 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.017890:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435038edf80/2, rc 0, transno 4294976112 00000100:00100000:2.0:1783520936.017896:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57d80 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.017906:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435038edf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993187712:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.017907:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006a0dc80a/0xd5401cbabc020d31 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x73a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16f2 expref: 461 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.017920:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d8a80 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.017920:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44617 00000100:00100000:2.0:1783520936.017933:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+543:3628:x1870152993189504:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.017943:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993190016:192.168.204.33@tcp:104: 00002000:00080000:2.0:1783520936.017948:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1849 count 1 00010000:00010000:2.0:1783520936.017962:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0260a5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x739:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.017967:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514dd5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993190144:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.017976:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.017980:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000eeb5b275/0xd5401cbabc020d2a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x739:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f16e4 expref: 461 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.017985:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x739:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.017988:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x739:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.017991:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0260a5 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x739:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.017995:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02606d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x731:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.018006:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.018009:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000eeb5b275/0xd5401cbabc020d2a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x739:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16e4 expref: 461 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.018013:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02606d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x731:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.018020:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000eeb5b275/0xd5401cbabc020d2a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x739:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16e4 expref: 461 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.018038:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993190144 00000100:00100000:2.0:1783520936.018043:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511adb480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993190272:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.018045:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993190144 00010000:00010000:1.0:1783520936.018046:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02606d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x731:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.018055:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02606d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x731:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.018066:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967a80 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.018069:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.018074:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44618 00000100:00100000:0.0:1783520936.018083:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514dd6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+544:3625:x1870152993190144:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.018097:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00002000:00080000:0.0:1783520936.018098:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1851 count 1 00010000:00010000:3.0:1783520936.018104:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02606d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x731:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.018114:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.018115:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0260ac lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x73b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.018120:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02606d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x731:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.018133:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b5ce5f77/0xd5401cbabc020d38 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x73b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1700 expref: 461 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.018139:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02606d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x731:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.018146:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x73b:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.018153:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x73b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.018158:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0260ac lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x73b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.018159:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02606d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x731:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.018161:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435038ed180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+545:3627:x1870152993187456:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6158us (6256us total) trans 4294976113 rc 0/0 00000100:00100000:1.0:1783520936.018180:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44609 00010000:00010000:2.0:1783520936.018181:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026058 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x72f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.018186:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02606d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x731:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.018201:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b5ce5f77/0xd5401cbabc020d38 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x73b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1700 expref: 461 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.018202:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026058 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x72f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.018211:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435038ee300/2, rc 0, transno 4294976113 00010000:00010000:3.0:1783520936.018213:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.018217:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.018222:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.018227:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026058 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x72f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.018227:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435038ee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993187456:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.018233:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b5ce5f77/0xd5401cbabc020d38 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x73b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1700 expref: 461 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.018247:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026058 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x72f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.018257:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514dd5f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993190400:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.018269:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.018279:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520936.018285:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993190528:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.018323:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02607b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x734:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.018330:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02607b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x734:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.018338:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.018339:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02607b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x734:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.018347:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026058 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x72f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.018349:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02607b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x734:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.018386:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520936.018388:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635786850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+545:3625:x1870152993187200:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6648us (6726us total) trans 4294976114 rc 0/0 00010000:00010000:3.0:1783520936.018395:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026058 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x72f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.018411:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435038ee680/2, rc 0, transno 4294976114 00000100:00100000:2.0:1783520936.018420:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44606 00010000:00010000:3.0:1783520936.018424:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026058 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x72f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.018427:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435038ee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993187200:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.018435:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.018449:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026058 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x72f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.018464:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026058 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x72f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.018485:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514dd5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993190656:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.018490:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.018507:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02607b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x734:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.018515:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.018520:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02607b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x734:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.018529:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02607b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x734:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.018539:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.018543:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02607b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x734:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.018547:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.018553:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02607b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x734:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.018555:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435080d3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+545:3626:x1870152993188096:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6265us (6442us total) trans 4294976115 rc 0/0 00000100:00100000:2.0:1783520936.018568:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993190528 00000100:00100000:1.0:1783520936.018573:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44611 00000100:00100000:2.0:1783520936.018575:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993190528 00010000:00010000:3.0:1783520936.018587:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.018592:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.018596:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520936.018597:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351dc73800/2, rc 0, transno 4294976115 00000100:00100000:1.0:1783520936.018602:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993190656 00000100:00100000:1.0:1783520936.018607:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993190656 00000100:00100000:2.0:1783520936.018612:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44619 00000100:00100000:0.0:1783520936.018613:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351dc73800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993188096:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.018625:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514dd4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+546:3626:x1870152993190528:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.018640:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1852 count 1 00000100:00100000:1.0:1783520936.018640:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44620 00000100:00100000:1.0:1783520936.018653:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514dd4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+546:3627:x1870152993190656:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.018656:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0260b3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x73c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.018659:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00002000:00080000:1.0:1783520936.018668:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1853 count 1 00010000:00010000:2.0:1783520936.018670:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003c1bccca/0xd5401cbabc020d46 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x73c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f170e expref: 461 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.018673:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514dd6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993190784:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.018684:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x73c:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.018685:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0260ba lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x73d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.018687:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x73c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.018696:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0260b3 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x73c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.018708:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.018709:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000000e6d43f0/0xd5401cbabc020d4d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x73d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f171c expref: 461 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.018720:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x73d:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.018724:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x73d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.018728:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0260ba lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x73d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.018730:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003c1bccca/0xd5401cbabc020d46 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x73c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f170e expref: 461 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.018754:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003c1bccca/0xd5401cbabc020d46 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x73c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f170e expref: 461 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.018769:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ad9500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993190912:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.018770:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000000e6d43f0/0xd5401cbabc020d4d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x73d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f171c expref: 461 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.018774:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff9436298a6480 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.018789:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000000e6d43f0/0xd5401cbabc020d4d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x73d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f171c expref: 461 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.018795:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993190784 00000100:00100000:0.0:1783520936.018801:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993190784 00000100:00100000:2.0:1783520936.018805:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c369480 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.018815:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9a680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993191040:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.018834:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44621 00000100:00100000:2.0:1783520936.018844:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514dd4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+547:3627:x1870152993190784:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.018852:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3eb40 going to sleep for 11 seconds 00002000:00080000:2.0:1783520936.018859:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1854 count 1 00010000:00010000:2.0:1783520936.018874:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0260c1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x73e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.018887:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002f09ac2e/0xd5401cbabc0228e4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x73e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fd4 expref: 461 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.018894:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x73e:0x0].0x0 since=36s v=(0 0) 00010000:00010000:2.0:1783520936.018897:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x73e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.018901:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0260c1 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x73e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.018924:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002f09ac2e/0xd5401cbabc0228e4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x73e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fd4 expref: 461 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.018935:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002f09ac2e/0xd5401cbabc0228e4 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x73e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fd4 expref: 461 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.018945:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ad8000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993191168:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.018981:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c3696c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.020595:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.020623:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993190016:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.020632:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.020655:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002437120 00010000:00010000:1.0:1783520936.020661:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc02609e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x73a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.020662:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.020662:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002437120 00000100:00100000:1.0:1783520936.020691:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002437248 00010000:00010000:0.0:1783520936.020692:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a95b3e11/0xd5401cbabc020d15 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x737:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16c8 expref: 462 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.020696:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002437248 00000100:00100000:2.0:1783520936.020711:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.020719:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000020646d0a/0xd5401cbabc020d23 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x738:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16d6 expref: 463 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.020727:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511adad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993189760:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.020735:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002437504 00000100:00100000:0.0:1783520936.020740:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002437504 00010000:00010000:3.0:1783520936.020753:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026090 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x738:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.020754:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002437376 00000100:00100000:1.0:1783520936.020760:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002437376 00010000:00010000:0.0:1783520936.020761:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000eeb5b275/0xd5401cbabc020d2a lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x739:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16e4 expref: 464 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.020794:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006a0dc80a/0xd5401cbabc020d31 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x73a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16f2 expref: 465 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.020802:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36205 00000100:00100000:3.0:1783520936.020815:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36206 00000100:00100000:0.0:1783520936.020817:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad8a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+465:2390:x1870153002437120:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520936.020830:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad9180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+465:2389:x1870153002437248:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.020831:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.020836:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36207 00010000:00010000:0.0:1783520936.020841:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a95b3e11/0xd5401cbabc020d15 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x737:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16c8 expref: 465 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.020845:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.020848:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36208 00000100:00100000:1.0:1783520936.020852:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+465:2390:x1870153002437504:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.020858:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000020646d0a/0xd5401cbabc020d23 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x738:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16d6 expref: 465 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.020865:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511adbb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+465:2389:x1870153002437376:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.020869:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.020870:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a95b3e11/0xd5401cbabc020d15 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x737:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16c8 expref: 465 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.020878:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.020878:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000eeb5b275/0xd5401cbabc020d2a lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x739:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16e4 expref: 465 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.020889:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006a0dc80a/0xd5401cbabc020d31 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x73a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16f2 expref: 465 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.020898:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x737:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.020900:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000eeb5b275/0xd5401cbabc020d2a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x739:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16e4 expref: 465 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.020905:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x737:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.020906:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006a0dc80a/0xd5401cbabc020d31 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x73a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16f2 expref: 465 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.020916:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026097 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x737:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.020934:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x739:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.020940:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026097 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x737:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.020941:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000020646d0a/0xd5401cbabc020d23 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x738:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16d6 expref: 465 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.020941:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x739:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.020944:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006a0dc80a/0xd5401cbabc020d31 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x73a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16f2 expref: 465 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.020952:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0260a5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x739:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.020975:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x73a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.020976:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0260a5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x739:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.020980:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.020983:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000020646d0a/0xd5401cbabc020d23 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x738:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16d6 expref: 464 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.020983:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x73a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.020994:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc02609e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.021004:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc02609e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x73a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.021006:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.021028:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.021080:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad8a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+464:2390:x1870153002437120:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 267us (487us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.021089:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36205 00000100:00100000:1.0:1783520936.021310:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+463:2390:x1870153002437504:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 463us (647us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.021322:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36207 00000100:00100000:2.0:1783520936.021325:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511adbb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+463:2389:x1870153002437376:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 462us (611us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.021336:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36208 00010000:00010000:3.0:1783520936.021348:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x738:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.021354:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x738:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.021381:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc02609e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.021383:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026090 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x738:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.021408:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026090 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x738:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.021408:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc02609e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.021411:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514dd5f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993190400:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.021436:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc02609e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.021438:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0260ac lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x73b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.021446:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.021450:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc02609e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x73a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.021471:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad9180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+460:2389:x1870153002437248:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 645us (838us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.021480:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36206 00010000:00010000:1.0:1783520936.021484:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.021503:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.021509:0:178496:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000eeb5b275/0xd5401cbabc020d2a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x739:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16e4 expref: 459 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.021512:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc02609e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x73a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.021517:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026090 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x738:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.021527:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511adb480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993190272:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.021543:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026090 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x738:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.021547:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0260a5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x739:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.021549:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.021556:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc02609e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x73a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.021565:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026090 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x738:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.021574:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0260a5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x739:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.021582:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc02609e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x73a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.021590:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026090 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x738:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.021598:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0260a5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x739:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.021612:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc02609e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x73a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.021621:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0260a5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x739:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.021623:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.021623:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc02609e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x73a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.021626:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+548:3628:x1870152993189632:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3880us (4005us total) trans 4294976116 rc 0/0 00010000:00010000:3.0:1783520936.021636:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.021645:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026090 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x738:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.021645:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44616 00010000:00010000:2.0:1783520936.021648:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0260a5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x739:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.021662:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.021666:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.021671:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.021672:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.021678:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026090 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x738:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.021680:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000004:00080000:1.0:1783520936.021689:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435161c1c00/2, rc 0, transno 4294976116 00010000:00010000:3.0:1783520936.021692:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026090 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x738:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.021694:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.021702:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0260a5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x739:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.021706:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993189632:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.021709:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.021720:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026090 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x738:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.021733:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.021738:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0260a5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x739:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.021748:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026090 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x738:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.021753:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0260a5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x739:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.021773:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0260a5 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x739:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.021775:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.021780:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.021783:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0260a5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x739:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.021785:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.021791:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.021798:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e98a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993191296:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.021809:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.021813:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.021816:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.021828:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.021837:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.021865:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.021895:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002437632 00000100:00100000:0.0:1783520936.021899:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002437632 00000100:00100000:1.0:1783520936.021900:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002437760 00000100:00100000:1.0:1783520936.021906:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002437760 00010000:00010000:0.0:1783520936.021919:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b5ce5f77/0xd5401cbabc020d38 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x73b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1700 expref: 460 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.021920:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+549:3628:x1870152993189504:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3988us (4238us total) trans 4294976117 rc 0/0 00010000:00010000:1.0:1783520936.021927:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003c1bccca/0xd5401cbabc020d46 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x73c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f170e expref: 460 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.021932:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a83100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+549:3626:x1870152993189376:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4312us (4589us total) trans 4294976118 rc 0/0 00000100:00100000:2.0:1783520936.021938:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44617 00000100:00100000:3.0:1783520936.021950:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44614 00000100:00100000:2.0:1783520936.021960:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993191296 00000100:00100000:2.0:1783520936.021964:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993191296 00000100:00100000:0.0:1783520936.021968:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002437888 00000100:00100000:0.0:1783520936.021973:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002437888 00010000:00010000:3.0:1783520936.021991:0:12734:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a95b3e11/0xd5401cbabc020d15 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x737:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f16c8 expref: 461 pid: 35275 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.021993:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002f09ac2e/0xd5401cbabc0228e4 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x73e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fd4 expref: 461 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.021998:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44622 00000004:00080000:1.0:1783520936.021999:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943509a83480/2, rc 0, transno 4294976118 00000100:00100000:2.0:1783520936.022013:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+548:3625:x1870152993191296:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.022016:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a83480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993189376:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.022018:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993189888:192.168.204.33@tcp:104: 00002000:00080000:2.0:1783520936.022034:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1855 count 1 00010000:00010000:3.0:1783520936.022037:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026097 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x737:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.022045:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026097 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x737:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.022047:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002438016 00000100:00100000:0.0:1783520936.022053:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002438016 00010000:00010000:3.0:1783520936.022054:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026097 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x737:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.022054:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0260c8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x73f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.022068:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:0.0:1783520936.022073:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000000e6d43f0/0xd5401cbabc020d4d lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x73d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f171c expref: 461 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.022076:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026097 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x737:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.022082:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e98700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993191424:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.022083:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000054b90393/0xd5401cbabc020d5b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x73f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1738 expref: 461 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.022109:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x73f:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.022112:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026097 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x737:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.022113:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36209 00010000:00010000:2.0:1783520936.022116:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x73f:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.022124:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.022127:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0260c8 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x73f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.022130:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514dd7100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+461:2388:x1870153002437632:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.022138:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.022145:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.022149:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.022153:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b5ce5f77/0xd5401cbabc020d38 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x73b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1700 expref: 461 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.022154:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026097 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x737:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.022162:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000054b90393/0xd5401cbabc020d5b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x73f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1738 expref: 461 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.022171:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435161c3b80/2, rc 0, transno 4294976117 00010000:00010000:3.0:1783520936.022177:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.022185:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026097 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x737:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.022186:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993189504:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.022187:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b5ce5f77/0xd5401cbabc020d38 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x73b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1700 expref: 461 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.022189:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000054b90393/0xd5401cbabc020d5b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x73f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1738 expref: 461 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.022211:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026097 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x737:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.022220:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511adb480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993191552:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.022225:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b5ce5f77/0xd5401cbabc020d38 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x73b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1700 expref: 461 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.022229:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026097 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x737:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.022244:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026097 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x737:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.022247:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x73b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.022255:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x73b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.022260:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0260ac lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.022261:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.022264:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:3.0:1783520936.022265:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.022269:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.022273:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895f00 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.022275:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0260ac lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x73b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.022276:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993191680:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.022286:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36210 00000100:00100000:2.0:1783520936.022298:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36211 00000100:00100000:3.0:1783520936.022299:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357c050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+460:2388:x1870153002437760:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.022309:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.022315:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.022317:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003c1bccca/0xd5401cbabc020d46 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x73c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f170e expref: 460 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.022325:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.022327:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003c1bccca/0xd5401cbabc020d46 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x73c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f170e expref: 460 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.022339:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x73c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.022341:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x73c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.022345:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0260b3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.022350:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36212 00000100:00100000:0.0:1783520936.022352:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514dd7100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+460:2388:x1870153002437632:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 226us (647us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.022386:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0260b3 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x73c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.022387:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514dd7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+460:2388:x1870153002438016:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.022396:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36209 00010000:00010000:1.0:1783520936.022403:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.022409:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.022411:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000000e6d43f0/0xd5401cbabc020d4d lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x73d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f171c expref: 459 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.022427:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000000e6d43f0/0xd5401cbabc020d4d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x73d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f171c expref: 459 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.022427:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993191424 00000100:00100000:3.0:1783520936.022428:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357c050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+459:2388:x1870153002437760:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 131us (640us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.022434:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993191424 00000100:00100000:3.0:1783520936.022441:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36210 00010000:00010000:1.0:1783520936.022455:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x73d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.022460:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x73d:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.022463:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993191680 00010000:00010000:1.0:1783520936.022465:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0260ba lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.022469:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993191680 00010000:00010000:1.0:1783520936.022474:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0260ba lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x73d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.022487:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.022508:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44623 00000100:00100000:0.0:1783520936.022516:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44624 00000100:00100000:3.0:1783520936.022522:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+550:3625:x1870152993191424:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.022522:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514dd7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+458:2388:x1870153002438016:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 161us (658us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.022531:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+550:3628:x1870152993191680:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.022536:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36212 00002000:00080000:3.0:1783520936.022541:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1856 count 1 00002000:00080000:0.0:1783520936.022550:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1857 count 1 00010000:00010000:3.0:1783520936.022556:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0260cf lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x740:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.022567:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0260d6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x741:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.022571:0:20415:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000e6d43f0/0xd5401cbabc020d4d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x73d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f171c expref: 457 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.022574:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003eb2df15/0xd5401cbabc020d62 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x740:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1746 expref: 457 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.022596:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x740:0x0].0x0 since=37s v=(0 0) 00000100:00100000:1.0:1783520936.022599:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9a680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993191040:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.022600:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003f399d40/0xd5401cbabc020d69 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x741:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1754 expref: 456 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.022601:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x740:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.022612:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0260cf lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x740:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.022617:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x741:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.022622:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0260ba lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.022626:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x741:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.022634:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0260d6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x741:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.022638:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003eb2df15/0xd5401cbabc020d62 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x740:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1746 expref: 456 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.022639:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0260ba lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.022662:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0260ba lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.022665:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003eb2df15/0xd5401cbabc020d62 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x740:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1746 expref: 456 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.022673:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003f399d40/0xd5401cbabc020d69 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x741:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1754 expref: 456 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.022679:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0260ba lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.022694:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993191808:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.022701:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0260ba lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x73d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.022707:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003f399d40/0xd5401cbabc020d69 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x741:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1754 expref: 456 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.022719:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.022725:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514dd5180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993191936:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.022734:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d83c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.022738:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:2.0:1783520936.022739:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514dd6300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+456:2389:x1870153002437888:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.022745:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0260ba lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x73d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.022751:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.022755:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.022760:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002f09ac2e/0xd5401cbabc0228e4 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x73e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fd4 expref: 456 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.022762:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0260ba lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x73d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.022783:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0260ba lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x73d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.022784:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002f09ac2e/0xd5401cbabc0228e4 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x73e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fd4 expref: 456 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.022801:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514dd4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+551:3627:x1870152993190656:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4148us (4262us total) trans 4294976119 rc 0/0 00000100:00100000:1.0:1783520936.022813:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44620 00010000:00010000:0.0:1783520936.022814:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0260ba lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x73d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.022815:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x73e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.022823:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x73e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.022832:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0260c1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.022833:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0260ba lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x73d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.022851:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0260c1 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x73e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.022862:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.022866:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.022867:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.022869:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520936.022897:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514dd5500/2, rc 0, transno 4294976119 00000100:00100000:1.0:1783520936.022901:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c016cc0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.022902:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514dd6300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+456:2389:x1870153002437888:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 592us (1072us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.022914:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36211 00000100:00100000:0.0:1783520936.022914:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514dd5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993190656:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.022941:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0260ac lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.022951:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0260ac lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.022960:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0260ac lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.022965:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:2.0:1783520936.022968:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0260ac lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x73b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.022978:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514dd5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993192064:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.022985:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520936.022994:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c9b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+551:3626:x1870152993189248:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5306us (5534us total) trans 4294976120 rc 0/0 00010000:00010000:2.0:1783520936.022996:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.023003:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0260ac lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x73b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.023007:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44615 00000100:00100000:0.0:1783520936.023016:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.023018:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.023023:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0260ac lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x73b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.023032:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0260ac lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x73b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.023044:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0260ac lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x73b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.023052:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc0260ac lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x73b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.023063:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993192064 00000004:00080000:3.0:1783520936.023065:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943509a80e00/2, rc 0, transno 4294976120 00000100:00100000:0.0:1783520936.023069:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993192064 00010000:00010000:2.0:1783520936.023071:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.023074:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.023077:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.023082:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a80e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993189248:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.023111:0:9151:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002f09ac2e/0xd5401cbabc0228e4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x73e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fd4 expref: 455 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.023127:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514dd7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993192192:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.023129:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ad8000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993191168:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.023147:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0260c1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.023156:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0260c1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.023169:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0260c1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.023176:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514dd6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+552:3625:x1870152993190144:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5092us (5172us total) trans 4294976121 rc 0/0 00000100:00100000:0.0:1783520936.023176:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.023186:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0260c1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.023190:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44618 00010000:00010000:1.0:1783520936.023199:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0260c1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x73e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.023212:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514dd5c00/2, rc 0, transno 4294976121 00000100:00100000:2.0:1783520936.023213:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993192192 00010000:00010000:1.0:1783520936.023218:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.023219:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993192192 00000100:00100000:0.0:1783520936.023228:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514dd5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993190144:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.023230:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.023237:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0260c1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x73e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.023252:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184a00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520936.023253:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44625 00010000:00010000:3.0:1783520936.023259:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0260c1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x73e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.023267:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514dd7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+553:3625:x1870152993192192:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.023275:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0260c1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x73e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.023278:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00002000:00080000:2.0:1783520936.023284:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1859 count 1 00000100:00100000:0.0:1783520936.023290:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514dd5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993192320:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.023296:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0260c1 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x73e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.023301:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0260dd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x743:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.023309:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0260c1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x73e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.023309:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514dd4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+553:3627:x1870152993190784:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4465us (4602us total) trans 4294976122 rc 0/0 00010000:00010000:2.0:1783520936.023320:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f3109a0d/0xd5401cbabc020d77 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x743:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1770 expref: 454 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.023326:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.023328:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44621 00010000:00010000:2.0:1783520936.023338:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x743:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.023343:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x743:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.023347:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0260dd lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x743:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.023349:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.023352:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993192320 00010000:00010000:3.0:1783520936.023354:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.023385:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993192320 00010000:00010000:3.0:1783520936.023403:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520936.023403:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514dd6a00/2, rc 0, transno 4294976122 00010000:00010000:2.0:1783520936.023404:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f3109a0d/0xd5401cbabc020d77 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x743:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1770 expref: 454 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.023420:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44626 00010000:00010000:2.0:1783520936.023421:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f3109a0d/0xd5401cbabc020d77 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x743:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1770 expref: 454 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.023421:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514dd6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993190784:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.023432:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516310700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+553:3625:x1870152993192320:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.023443:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ad9880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993192448:192.168.204.33@tcp:104: 00002000:00080000:1.0:1783520936.023449:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1860 count 1 00010000:00010000:1.0:1783520936.023464:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0260e4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x744:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.023475:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000059caa99b/0xd5401cbabc020d7e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x744:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f177e expref: 454 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.023482:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516311500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993192576:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.023484:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x744:0x0].0x0 since=37s v=(0 0) 00000100:00100000:2.0:1783520936.023501:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895900 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.023501:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x744:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.023510:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0260e4 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x744:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.023528:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.023535:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000059caa99b/0xd5401cbabc020d7e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x744:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f177e expref: 454 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.023547:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000059caa99b/0xd5401cbabc020d7e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x744:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f177e expref: 454 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.023550:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993192576 00000100:00100000:2.0:1783520936.023557:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993192576 00000100:00100000:1.0:1783520936.023561:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a6300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993192704:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.023593:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.023597:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44627 00000100:00100000:2.0:1783520936.023606:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516311880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+554:3627:x1870152993192576:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.023612:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44628 00002000:00080000:2.0:1783520936.023623:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1861 count 1 00000100:00100000:0.0:1783520936.023623:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514dd6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+554:3627:x1870152993192064:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.023640:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0260eb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x745:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.023641:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1858 count 1 00010000:00010000:0.0:1783520936.023658:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0260f2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x742:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.023662:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c19c8d3b/0xd5401cbabc020d85 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x745:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f178c expref: 454 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.023675:0:8410:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003c1bccca/0xd5401cbabc020d46 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x73c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f170e expref: 454 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.023676:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001bfc1dea/0xd5401cbabc020d70 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x742:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1762 expref: 454 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.023704:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x745:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.023704:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x742:0x0].0x0 since=37s v=(0 0) 00000100:00100000:1.0:1783520936.023705:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ad9500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993190912:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.023714:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x745:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.023714:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x742:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.023725:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0260eb lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x745:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.023728:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0260b3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.023728:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0260f2 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x742:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.023755:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0260b3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.023760:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c19c8d3b/0xd5401cbabc020d85 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x745:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f178c expref: 453 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.023771:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0260b3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.023776:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001bfc1dea/0xd5401cbabc020d70 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x742:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1762 expref: 453 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.023779:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c19c8d3b/0xd5401cbabc020d85 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x745:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f178c expref: 453 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.023807:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001bfc1dea/0xd5401cbabc020d70 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x742:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1762 expref: 453 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.023808:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0260b3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.023809:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511adaa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993192832:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.023833:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0260b3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x73c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.023837:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516313800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993192960:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.023851:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.023853:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895c00 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.023862:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.023869:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0260b3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x73c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.023872:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435122b6f00 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.023881:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.023885:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0260b3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x73c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.023895:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0260b3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x73c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.023909:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0260b3 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x73c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.023917:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0260b3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x73c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.023934:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514dd4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+555:3626:x1870152993190528:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5311us (5596us total) trans 4294976123 rc 0/0 00000100:00100000:1.0:1783520936.023943:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44619 00000004:00080000:0.0:1783520936.023961:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514ff0e00/2, rc 0, transno 4294976123 00010000:00010000:2.0:1783520936.023966:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.023969:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.023972:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.023974:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993190528:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.024035:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516310e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993193088:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.024063:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.024098:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993193088 00000100:00100000:0.0:1783520936.024101:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993193088 00000100:00100000:0.0:1783520936.024125:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44629 00000100:00100000:0.0:1783520936.024133:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+555:3628:x1870152993193088:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.024145:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1862 count 1 00010000:00010000:0.0:1783520936.024156:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0260f9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x746:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.024168:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c8726385/0xd5401cbabc022b3e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x746:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f322e expref: 453 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.024174:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x746:0x0].0x0 since=36s v=(0 0) 00010000:00010000:0.0:1783520936.024177:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x746:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.024182:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0260f9 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x746:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.024200:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c8726385/0xd5401cbabc022b3e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x746:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f322e expref: 453 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.024209:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c8726385/0xd5401cbabc022b3e lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x746:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f322e expref: 453 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.024217:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516312a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993193216:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.024249:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435192190c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.025829:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511adb480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993191552:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.025857:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0260c8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x73f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.025863:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.025892:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.025899:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993191808:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.025901:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ad9880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993192448:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.025916:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.025927:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0260cf lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x740:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.025929:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0260dd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x743:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.025942:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.025965:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002438144 00000100:00100000:3.0:1783520936.025968:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002438400 00000100:00100000:1.0:1783520936.025971:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002438144 00000100:00100000:3.0:1783520936.025974:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002438400 00010000:00010000:1.0:1783520936.025998:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000054b90393/0xd5401cbabc020d5b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x73f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1738 expref: 455 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.026026:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002438528 00000100:00100000:0.0:1783520936.026027:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514dd5180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993191936:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.026030:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002438528 00010000:00010000:1.0:1783520936.026044:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f3109a0d/0xd5401cbabc020d77 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x743:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1770 expref: 456 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.026047:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0260d6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x741:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.026082:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002438272 00000100:00100000:1.0:1783520936.026085:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002438272 00000100:00100000:0.0:1783520936.026088:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36213 00010000:00010000:1.0:1783520936.026099:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003eb2df15/0xd5401cbabc020d62 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x740:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1746 expref: 457 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.026107:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a5500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+457:2390:x1870153002438144:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.026120:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.026126:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000054b90393/0xd5401cbabc020d5b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x73f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1738 expref: 457 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.026131:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36214 00000100:00100000:1.0:1783520936.026143:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a5180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+457:2390:x1870153002438528:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.026147:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000054b90393/0xd5401cbabc020d5b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x73f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1738 expref: 457 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.026153:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.026161:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36215 00010000:00010000:1.0:1783520936.026161:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f3109a0d/0xd5401cbabc020d77 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x743:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1770 expref: 457 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.026169:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000054b90393/0xd5401cbabc020d5b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x73f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1738 expref: 457 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.026178:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a7b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+457:2389:x1870153002438272:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.026192:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f3109a0d/0xd5401cbabc020d77 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x743:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1770 expref: 457 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.026195:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.026198:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x73f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.026206:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x73f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.026212:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0260c8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.026214:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f3109a0d/0xd5401cbabc020d77 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x743:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1770 expref: 456 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.026220:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003eb2df15/0xd5401cbabc020d62 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x740:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1746 expref: 456 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.026237:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0260c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x73f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.026242:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x743:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.026249:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003eb2df15/0xd5401cbabc020d62 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x740:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1746 expref: 455 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.026251:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x743:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.026260:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0260dd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x743:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.026270:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0260dd lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x743:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.026270:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.026273:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003eb2df15/0xd5401cbabc020d62 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x740:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1746 expref: 455 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.026309:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a5500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+455:2390:x1870153002438144:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 204us (446us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.026316:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x740:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.026320:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36213 00010000:00010000:1.0:1783520936.026322:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.026323:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x740:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.026345:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a5180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+453:2390:x1870153002438528:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 205us (431us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.026347:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0260c8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.026379:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36214 00010000:00010000:0.0:1783520936.026404:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0260c8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.026421:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0260c8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x73f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.026429:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0260c8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x73f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.026430:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a6300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993192704:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.026451:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.026452:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0260e4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x744:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.026468:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.026475:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0260c8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x73f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.026485:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.026489:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0260cf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x740:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.026490:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0260c8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x73f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.026522:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.026524:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0260cf lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x740:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.026526:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0260c8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x73f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.026541:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0260c8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x73f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.026549:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0260c8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x73f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.026569:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.026571:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.026574:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.026583:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002438656 00000100:00100000:0.0:1783520936.026587:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002438656 00010000:00010000:0.0:1783520936.026603:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000059caa99b/0xd5401cbabc020d7e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x744:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f177e expref: 453 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.026621:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.026631:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36216 00000100:00100000:0.0:1783520936.026641:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357cc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+453:2389:x1870153002438656:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.026652:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.026657:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.026664:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a7b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+453:2389:x1870153002438272:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 490us (723us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.026666:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000059caa99b/0xd5401cbabc020d7e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x744:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f177e expref: 453 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.026672:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002438784 00000100:00100000:3.0:1783520936.026677:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36215 00000100:00100000:1.0:1783520936.026678:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002438784 00010000:00010000:0.0:1783520936.026693:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000059caa99b/0xd5401cbabc020d7e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x744:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f177e expref: 454 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.026698:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c19c8d3b/0xd5401cbabc020d85 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x745:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f178c expref: 454 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.026716:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000059caa99b/0xd5401cbabc020d7e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x744:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f177e expref: 454 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.026726:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36217 00010000:00010000:0.0:1783520936.026728:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x744:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.026732:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x744:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.026737:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a6300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+452:2390:x1870153002438784:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.026740:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0260e4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x744:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.026748:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.026750:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0260e4 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x744:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.026756:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c19c8d3b/0xd5401cbabc020d85 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x745:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f178c expref: 452 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.026770:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c19c8d3b/0xd5401cbabc020d85 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x745:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f178c expref: 452 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.026775:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.026796:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x745:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.026798:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357cc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+452:2389:x1870153002438656:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 158us (279us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.026800:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x745:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.026810:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0260eb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x745:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.026811:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36216 00010000:00010000:1.0:1783520936.026821:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0260eb lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x745:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.026836:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.026839:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.026846:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516312a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993193216:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.026858:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a6300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+451:2390:x1870153002438784:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 123us (208us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.026866:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36217 00010000:00010000:0.0:1783520936.026872:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0260f9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x746:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.026881:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002439040 00000100:00100000:3.0:1783520936.026882:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.026887:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002439040 00010000:00010000:1.0:1783520936.026904:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c8726385/0xd5401cbabc022b3e lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x746:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f322e expref: 451 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.026912:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002438912 00000100:00100000:3.0:1783520936.026918:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002438912 00000100:00100000:0.0:1783520936.026921:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516313800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993192960:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.026931:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36218 00010000:00010000:3.0:1783520936.026941:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001bfc1dea/0xd5401cbabc020d70 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x742:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1762 expref: 452 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.026943:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350ade1f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+452:2390:x1870153002439040:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.026943:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0260f2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x742:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.026960:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.026968:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c8726385/0xd5401cbabc022b3e lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x746:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f322e expref: 452 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.026979:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36219 00010000:00010000:1.0:1783520936.026982:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c8726385/0xd5401cbabc022b3e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x746:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f322e expref: 452 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.026989:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350ade0700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+452:2389:x1870153002438912:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.027004:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.027008:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c8726385/0xd5401cbabc022b3e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x746:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f322e expref: 452 pid: 12734 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.027011:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001bfc1dea/0xd5401cbabc020d70 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x742:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1762 expref: 452 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.027035:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x746:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.027036:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001bfc1dea/0xd5401cbabc020d70 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x742:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1762 expref: 451 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.027041:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x746:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.027048:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0260f9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x746:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.027054:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001bfc1dea/0xd5401cbabc020d70 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x742:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1762 expref: 451 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.027054:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003f399d40/0xd5401cbabc020d69 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x741:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1754 expref: 451 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.027065:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0260f9 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x746:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.027078:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x742:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.027086:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x742:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.027089:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0260f2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x742:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.027095:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0260f2 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x742:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.027097:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.027112:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.027119:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0260f9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x746:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.027121:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350ade1f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+450:2390:x1870153002439040:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 181us (337us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.027135:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350ade0700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+450:2389:x1870153002438912:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 146us (256us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.027138:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0260f9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x746:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.027141:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36218 00000100:00100000:3.0:1783520936.027151:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36219 00010000:00010000:0.0:1783520936.027156:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0260f9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x746:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.027165:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0260f9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x746:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.027177:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0260cf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x740:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.027179:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0260e4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x744:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.027186:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.027195:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0260cf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x740:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.027198:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0260e4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x744:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.027206:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0260cf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x740:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.027214:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0260e4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x744:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.027215:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0260cf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x740:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.027238:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0260e4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x744:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.027241:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520936.027253:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1783520936.027258:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520936.027259:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0260f9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x746:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.027265:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.027267:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0260f9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x746:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.027272:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0260f9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x746:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.027281:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0260f9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x746:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.027286:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0260f9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x746:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.027299:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520936.027301:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0260cf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x740:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.027310:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.027312:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0260cf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x740:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.027317:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0260cf lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x740:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.027323:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0260cf lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x740:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.027327:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0260cf lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x740:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.027339:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+557:3628:x1870152993193088:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3204us (3272us total) trans 4294976124 rc 0/0 00010000:00010000:3.0:1783520936.027340:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.027345:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0260e4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x744:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.027351:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbaa00 already unused, calling callback (ffffffffc0bf9e90) 00000004:00080000:1.0:1783520936.027351:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516310e00/2, rc 0, transno 4294976124 00000100:00100000:0.0:1783520936.027354:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44629 00010000:00010000:3.0:1783520936.027372:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0260e4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x744:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.027378:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0260e4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x744:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.027390:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0260e4 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x744:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.027393:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516310e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993193088:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.027405:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0260e4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x744:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.027421:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.027425:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.027427:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.027437:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516310700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+556:3625:x1870152993192320:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4008us (4115us total) trans 4294976125 rc 0/0 00000100:00100000:1.0:1783520936.027445:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44626 00000100:00100000:0.0:1783520936.027450:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516313800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993193344:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.027479:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:0.0:1783520936.027520:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514dd5f80/2, rc 0, transno 4294976125 00000100:00100000:0.0:1783520936.027531:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514dd5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993192320:0@lo:6:osp-syn-1-0.0 00000004:00080000:0.0:1783520936.027550:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e98700/2, rc 0, transno 4294976126 00000100:00100000:3.0:1783520936.027556:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+556:3625:x1870152993191424:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5037us (5434us total) trans 4294976126 rc 0/0 00000100:00100000:0.0:1783520936.027560:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e98700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993191424:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.027569:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44623 00000100:00100000:3.0:1783520936.027587:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993193344 00000100:00100000:3.0:1783520936.027591:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993193344 00000100:00100000:3.0:1783520936.027623:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44630 00010000:00010000:1.0:1783520936.027625:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0260f2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x742:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.027635:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516312a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+556:3625:x1870152993193344:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.027636:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:1.0:1783520936.027638:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0260f2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x742:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.027646:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516313480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993193472:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.027656:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0260f2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x742:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.027665:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0260f2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x742:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.027671:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.027686:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.027699:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516310a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993193600:0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520936.027701:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1863 count 1 00000100:00100000:0.0:1783520936.027716:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.027719:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026100 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x747:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.027731:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000cc3a9a06/0xd5401cbabc020d8c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x747:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f17a8 expref: 448 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.027737:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x747:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.027738:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.027740:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x747:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.027743:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0260f2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x742:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.027744:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026100 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x747:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.027763:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.027767:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0260f2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x742:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.027772:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514dd6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+557:3627:x1870152993192064:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4150us (4759us total) trans 4294976127 rc 0/0 00010000:00010000:3.0:1783520936.027775:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000cc3a9a06/0xd5401cbabc020d8c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x747:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17a8 expref: 448 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.027781:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0260f2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x742:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.027789:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44628 00010000:00010000:0.0:1783520936.027802:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0260f2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x742:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.027806:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993193472 00010000:00010000:3.0:1783520936.027811:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000cc3a9a06/0xd5401cbabc020d8c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x747:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17a8 expref: 448 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.027812:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993193472 00010000:00010000:0.0:1783520936.027816:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0260f2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x742:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.027831:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993193600 00000100:00100000:1.0:1783520936.027837:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993193600 00000100:00100000:3.0:1783520936.027841:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993193728:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.027847:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.027850:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.027854:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.027857:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44631 00000100:00100000:1.0:1783520936.027866:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516310000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+558:3625:x1870152993193472:12345-0@lo:6:osp-syn-1-0.0 00000004:00080000:0.0:1783520936.027872:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514dd5880/2, rc 0, transno 4294976127 00002000:00080000:1.0:1783520936.027879:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1864 count 1 00000100:00100000:0.0:1783520936.027882:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514dd5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993192064:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.027895:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026107 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x748:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.027900:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d83c0 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.027911:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000de8d0ccd/0xd5401cbabc020d93 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x748:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f17b6 expref: 448 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.027919:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x748:0x0].0x0 since=37s v=(0 0) 00000100:00100000:0.0:1783520936.027920:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:1.0:1783520936.027926:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x748:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.027930:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516311f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993193856:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.027933:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026107 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x748:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.027951:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.027958:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000de8d0ccd/0xd5401cbabc020d93 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x748:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17b6 expref: 448 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.027970:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000de8d0ccd/0xd5401cbabc020d93 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x748:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17b6 expref: 448 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.027978:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993193856 00000100:00100000:1.0:1783520936.027982:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a6a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993193984:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.027983:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993193856 00000100:00100000:3.0:1783520936.027991:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44632 00000100:00100000:3.0:1783520936.028002:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516311c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+559:3627:x1870152993193600:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.028009:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44633 00002000:00080000:3.0:1783520936.028018:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1865 count 1 00000100:00100000:1.0:1783520936.028020:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e240 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.028020:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516313b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+559:3627:x1870152993193856:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.028037:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1866 count 1 00010000:00010000:0.0:1783520936.028051:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02610e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x74a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.028064:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000094290c08/0xd5401cbabc020da8 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x74a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f17d2 expref: 448 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.028071:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x74a:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.028074:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026115 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x749:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.028074:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x74a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.028079:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02610e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x74a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.028092:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000082e13b5e/0xd5401cbabc020da1 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x749:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f17c4 expref: 448 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.028104:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x749:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.028108:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x749:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.028110:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+560:3625:x1870152993191296:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6100us (6273us total) trans 4294976128 rc 0/0 00010000:00010000:3.0:1783520936.028114:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026115 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x749:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.028123:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44622 00000004:00080000:0.0:1783520936.028124:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e98a80/2, rc 0, transno 4294976128 00000100:00100000:0.0:1783520936.028139:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e98a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993191296:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.028146:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000082e13b5e/0xd5401cbabc020da1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x749:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17c4 expref: 448 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.028147:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.028160:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000082e13b5e/0xd5401cbabc020da1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x749:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17c4 expref: 448 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.028173:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.028180:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade1880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993194112:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.028186:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1783520936.028192:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36220 00000100:00100000:0.0:1783520936.028200:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516310380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993194240:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.028230:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57d80 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.028235:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.028245:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000094290c08/0xd5401cbabc020da8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x74a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17d2 expref: 448 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.028257:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000094290c08/0xd5401cbabc020da8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x74a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17d2 expref: 448 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.028268:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993194368:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.028275:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993194240 00000100:00100000:0.0:1783520936.028279:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993194240 00000100:00100000:2.0:1783520936.028285:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a4700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+448:2388:x1870153002438400:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.028297:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.028298:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e3c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.028304:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44634 00010000:00010000:2.0:1783520936.028305:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003f399d40/0xd5401cbabc020d69 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x741:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1754 expref: 448 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.028314:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516312680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+560:3626:x1870152993194240:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.028327:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003f399d40/0xd5401cbabc020d69 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x741:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1754 expref: 448 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.028327:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1867 count 1 00010000:00010000:0.0:1783520936.028342:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02611c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x74b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.028346:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0260dd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x743:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.028349:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003f399d40/0xd5401cbabc020d69 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x741:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1754 expref: 448 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.028395:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000000870b2e8/0xd5401cbabc020daf lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x74b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f17e0 expref: 447 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.028397:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x741:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.028402:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0260dd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x743:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.028405:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x741:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.028416:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0260d6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x741:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.028419:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x74b:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.028428:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0260dd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x743:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.028429:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x74b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.028439:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02611c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x74b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.028456:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0260dd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x743:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.028460:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0260d6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x741:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.028472:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000000870b2e8/0xd5401cbabc020daf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x74b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17e0 expref: 447 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.028477:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.028481:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000000870b2e8/0xd5401cbabc020daf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x74b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17e0 expref: 447 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.028490:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514dd5180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993194496:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.028540:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c016cc0 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.028541:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.028548:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0260dd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x743:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.028554:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.028558:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.028559:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0260dd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x743:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.028567:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0260dd lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x743:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.028578:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0260dd lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x743:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.028584:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0260dd lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x743:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.028587:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a4700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+447:2388:x1870153002438400:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 384us (2692us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.028590:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514dd7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+561:3625:x1870152993192192:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5325us (5417us total) trans 4294976129 rc 0/0 00000004:00080000:0.0:1783520936.028604:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514dd7b80/2, rc 0, transno 4294976129 00000100:00100000:2.0:1783520936.028607:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36220 00000100:00100000:1.0:1783520936.028607:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44625 00000100:00100000:0.0:1783520936.028619:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514dd7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993192192:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.028663:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c19c8d3b/0xd5401cbabc020d85 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x745:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f178c expref: 446 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.028672:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503325f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993194624:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.028676:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0260d6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x741:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.028687:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.028690:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.028694:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.028697:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0260d6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x741:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.028702:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511adaa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993192832:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.028706:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0260d6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x741:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.028715:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.028718:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0260d6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x741:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.028720:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0260eb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x745:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.028733:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.028733:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0260eb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x745:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.028744:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0260eb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x745:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.028751:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.028753:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0260eb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x745:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.028756:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0260d6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x741:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.028767:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0260eb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x745:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.028768:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.028774:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0260d6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x741:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.028783:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0260d6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x741:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.028792:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.028792:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0260d6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x741:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.028801:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0260d6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x741:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.028803:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:2.0:1783520936.028810:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+561:3628:x1870152993191680:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6284us (6497us total) trans 4294976130 rc 0/0 00010000:00010000:3.0:1783520936.028812:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0260eb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x745:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.028817:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44624 00010000:00010000:0.0:1783520936.028818:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.028821:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.028825:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.028836:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185400 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520936.028837:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993194624 00010000:00010000:3.0:1783520936.028841:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0260eb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x745:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.028841:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993194624 00010000:00010000:3.0:1783520936.028850:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0260eb lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x745:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.028860:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0260eb lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x745:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.028866:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0260eb lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x745:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.028867:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44635 00000100:00100000:1.0:1783520936.028883:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516311880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+562:3627:x1870152993192576:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5277us (5354us total) trans 4294976131 rc 0/0 00010000:00010000:3.0:1783520936.028885:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.028888:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.028892:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.028893:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44627 00000004:00080000:0.0:1783520936.028898:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516311500/2, rc 0, transno 4294976131 00000100:00100000:0.0:1783520936.028909:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516311500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993192576:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.028929:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635786050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+561:3626:x1870152993194624:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.028942:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1868 count 1 00000004:00080000:2.0:1783520936.028947:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514ff1c00/2, rc 0, transno 4294976130 00000100:00100000:0.0:1783520936.028952:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503326a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993194752:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.028955:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026123 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x74c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.028964:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993191680:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.028972:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000040113e79/0xd5401cbabc020db6 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x74c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f17ee expref: 445 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.028984:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x74c:0x0].0x0 since=37s v=(0 0) 00000100:00100000:0.0:1783520936.028984:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.028989:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x74c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.028994:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026123 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x74c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.029010:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:1.0:1783520936.029017:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000040113e79/0xd5401cbabc020db6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x74c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17ee expref: 445 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.029019:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993194880:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.029033:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000040113e79/0xd5401cbabc020db6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x74c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17ee expref: 445 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.029045:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993195008:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.029051:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.029071:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993194752 00000100:00100000:3.0:1783520936.029075:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993194752 00000100:00100000:0.0:1783520936.029106:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993194880 00000100:00100000:3.0:1783520936.029109:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44636 00000100:00100000:0.0:1783520936.029112:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993194880 00000100:00100000:3.0:1783520936.029119:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503325880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+563:3627:x1870152993194752:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.029151:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44637 00000100:00100000:0.0:1783520936.029159:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503324700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+563:3627:x1870152993194880:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.029174:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1869 count 1 00002000:00080000:0.0:1783520936.029175:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1870 count 1 00010000:00010000:2.0:1783520936.029193:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02612a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x74d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.029194:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3eb40 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.029194:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026131 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x74e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.029222:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004d594477/0xd5401cbabc020dbd lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x74d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f17fc expref: 445 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.029223:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f6cd075f/0xd5401cbabc022a2d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x74e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f311d expref: 445 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.029242:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x74d:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.029245:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x74e:0x0].0x0 since=36s v=(0 0) 00010000:00010000:2.0:1783520936.029247:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x74d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.029250:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x74e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.029254:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02612a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x74d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.029256:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026131 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x74e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.029290:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004d594477/0xd5401cbabc020dbd lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x74d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17fc expref: 445 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.029296:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f6cd075f/0xd5401cbabc022a2d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x74e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f311d expref: 445 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.029307:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004d594477/0xd5401cbabc020dbd lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x74d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17fc expref: 445 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.029312:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f6cd075f/0xd5401cbabc022a2d lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x74e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f311d expref: 445 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.029327:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ad9880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993195136:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.029341:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503326680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993195264:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.029402:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435122b6f00 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.029404:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895f00 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.030527:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993193728:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.030544:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.030551:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026100 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x747:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.030564:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a6a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993193984:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.030582:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.030583:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026107 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x748:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.030612:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002439168 00000100:00100000:3.0:1783520936.030619:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002439168 00010000:00010000:3.0:1783520936.030650:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000cc3a9a06/0xd5401cbabc020d8c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x747:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17a8 expref: 446 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.030661:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993194368:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.030671:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade1880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993194112:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.030678:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002439296 00000100:00100000:3.0:1783520936.030682:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002439296 00010000:00010000:2.0:1783520936.030694:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026115 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x749:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.030696:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02610e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x74a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.030700:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000de8d0ccd/0xd5401cbabc020d93 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x748:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17b6 expref: 447 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.030741:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36221 00000100:00100000:3.0:1783520936.030751:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511adb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+447:2387:x1870153002439168:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.030762:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.030766:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36222 00010000:00010000:3.0:1783520936.030771:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000cc3a9a06/0xd5401cbabc020d8c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x747:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17a8 expref: 447 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.030784:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000cc3a9a06/0xd5401cbabc020d8c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x747:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17a8 expref: 447 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.030820:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cc3a9a06/0xd5401cbabc020d8c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x747:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17a8 expref: 447 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.030833:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x747:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.030834:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511adaa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+446:2388:x1870153002439296:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.030840:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x747:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.030846:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.030847:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026100 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x747:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.030854:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000de8d0ccd/0xd5401cbabc020d93 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x748:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17b6 expref: 446 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.030862:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026100 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x747:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.030869:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000de8d0ccd/0xd5401cbabc020d93 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x748:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17b6 expref: 446 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.030888:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000de8d0ccd/0xd5401cbabc020d93 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x748:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17b6 expref: 446 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.030899:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x748:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.030903:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x748:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.030907:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026107 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x748:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.030914:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026107 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x748:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.030940:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.030968:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026107 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x748:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.030979:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026107 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x748:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.030984:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511adaa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+445:2388:x1870153002439296:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 206us (401us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.030996:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026107 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x748:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.031001:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36222 00010000:00010000:0.0:1783520936.031008:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026107 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x748:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.031032:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.031039:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.031045:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026107 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x748:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.031054:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.031058:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026107 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x748:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.031068:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026107 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x748:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.031080:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026107 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x748:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.031088:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026107 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x748:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.031108:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.031111:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.031113:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.031156:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516310000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+564:3625:x1870152993193472:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3291us (3486us total) trans 4294976132 rc 0/0 00000100:00100000:0.0:1783520936.031166:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44631 00000004:00080000:2.0:1783520936.031186:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516313480/2, rc 0, transno 4294976132 00000100:00100000:2.0:1783520936.031201:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516313480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993193472:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.031203:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.031232:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511adb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+444:2387:x1870153002439168:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 483us (690us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.031242:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36221 00000100:00100000:0.0:1783520936.031263:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1783520936.031276:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503326300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993195392:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.031306:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.031348:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026100 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x747:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.031349:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.031399:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993195392 00000100:00100000:1.0:1783520936.031405:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993195392 00000100:00100000:1.0:1783520936.031440:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44638 00010000:00010000:3.0:1783520936.031443:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026100 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x747:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.031451:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503327800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+564:3626:x1870152993195392:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.031452:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026100 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x747:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.031464:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026100 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x747:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.031477:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1871 count 1 00000100:00100000:0.0:1783520936.031481:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514dd5180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993194496:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.031510:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026138 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x74f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.031513:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.031516:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02611c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x74b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.031536:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002439424 00010000:00010000:1.0:1783520936.031538:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000fa3c26b1/0xd5401cbabc020dcb lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x74f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1818 expref: 443 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.031542:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002439424 00010000:00010000:1.0:1783520936.031549:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x74f:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.031553:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x74f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.031555:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.031558:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026138 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x74f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.031564:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.031564:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000094290c08/0xd5401cbabc020da8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x74a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17d2 expref: 444 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.031590:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000fa3c26b1/0xd5401cbabc020dcb lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x74f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1818 expref: 444 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.031595:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002439680 00000100:00100000:2.0:1783520936.031600:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002439680 00010000:00010000:1.0:1783520936.031606:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000fa3c26b1/0xd5401cbabc020dcb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x74f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1818 expref: 445 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.031620:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000000870b2e8/0xd5401cbabc020daf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x74b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17e0 expref: 445 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.031622:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026100 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x747:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.031629:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993195520:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.031641:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516312a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+565:3625:x1870152993193344:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4008us (4158us total) trans 4294976133 rc 0/0 00010000:00010000:3.0:1783520936.031647:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4200 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520936.031652:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44630 00010000:00010000:3.0:1783520936.031653:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026100 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x747:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.031662:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36223 00000100:00100000:2.0:1783520936.031672:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+445:2388:x1870153002439424:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.031681:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000004:00080000:0.0:1783520936.031688:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516313800/2, rc 0, transno 4294976133 00010000:00010000:2.0:1783520936.031689:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000094290c08/0xd5401cbabc020da8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x74a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17d2 expref: 445 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.031689:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e3c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.031701:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516313800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993193344:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.031706:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000094290c08/0xd5401cbabc020da8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x74a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17d2 expref: 445 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.031714:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36224 00000100:00100000:1.0:1783520936.031727:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357e850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+445:2389:x1870153002439680:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.031730:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000094290c08/0xd5401cbabc020da8 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x74a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17d2 expref: 445 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.031740:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.031745:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x74a:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.031745:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503327480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993195648:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.031749:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000000870b2e8/0xd5401cbabc020daf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x74b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17e0 expref: 444 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.031754:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x74a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.031765:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02610e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x74a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.031769:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000000870b2e8/0xd5401cbabc020daf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x74b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17e0 expref: 444 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.031783:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.031784:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026100 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x747:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.031785:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02610e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x74a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.031800:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000870b2e8/0xd5401cbabc020daf lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x74b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17e0 expref: 444 pid: 8409 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.031816:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026100 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x747:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.031831:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x74b:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.031831:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503326680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993195264:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.031834:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.031841:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x74b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.031843:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026100 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x747:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.031849:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02611c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x74b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.031855:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+443:2388:x1870153002439424:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 185us (508us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.031856:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026131 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x74e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.031863:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.031868:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02611c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x74b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.031869:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.031872:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36223 00010000:00010000:3.0:1783520936.031873:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.031894:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993195648 00010000:00010000:1.0:1783520936.031896:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.031899:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993195648 00000100:00100000:2.0:1783520936.031912:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ad9880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993195136:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.031927:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02612a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x74d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.031927:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357e850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+442:2389:x1870153002439680:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 200us (414us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.031929:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.031936:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44639 00000100:00100000:1.0:1783520936.031947:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36224 00000100:00100000:0.0:1783520936.031951:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503327100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+565:3626:x1870152993195648:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.031966:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1872 count 1 00010000:00010000:0.0:1783520936.031981:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02613f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x750:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.031993:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d1a24206/0xd5401cbabc020dd9 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x750:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1826 expref: 441 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.031994:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002439808 00000100:00100000:1.0:1783520936.032000:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002439808 00000100:00100000:2.0:1783520936.032001:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.032009:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x750:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.032013:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x750:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.032017:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02613f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x750:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.032020:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f6cd075f/0xd5401cbabc022a2d lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x74e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f311d expref: 442 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.032021:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002439552 00000100:00100000:2.0:1783520936.032026:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002439552 00010000:00010000:2.0:1783520936.032045:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000082e13b5e/0xd5401cbabc020da1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x749:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17c4 expref: 443 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.032052:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d1a24206/0xd5401cbabc020dd9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x750:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1826 expref: 443 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.032059:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36225 00010000:00010000:0.0:1783520936.032073:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d1a24206/0xd5401cbabc020dd9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x750:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1826 expref: 443 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.032074:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350ade1c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+443:2387:x1870153002439808:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.032078:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36226 00000100:00100000:1.0:1783520936.032093:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+443:2387:x1870153002439552:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.032093:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503326680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993195776:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.032108:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.032114:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000082e13b5e/0xd5401cbabc020da1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x749:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17c4 expref: 443 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.032126:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000082e13b5e/0xd5401cbabc020da1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x749:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17c4 expref: 443 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.032134:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.032135:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435192190c0 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.032144:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f6cd075f/0xd5401cbabc022a2d lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x74e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f311d expref: 443 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.032144:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000082e13b5e/0xd5401cbabc020da1 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x749:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17c4 expref: 443 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.032151:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02611c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x74b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.032171:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02611c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x74b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.032175:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f6cd075f/0xd5401cbabc022a2d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x74e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f311d expref: 442 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.032175:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x749:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.032178:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02611c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x74b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.032183:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x749:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.032193:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026115 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x749:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.032202:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02611c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x74b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.032214:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026115 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x749:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.032217:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f6cd075f/0xd5401cbabc022a2d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x74e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f311d expref: 442 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.032240:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x74e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.032244:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x74e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.032244:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.032251:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026131 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x74e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.032262:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026115 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x749:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.032264:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026131 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x74e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.032282:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+441:2387:x1870153002439552:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 191us (283us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.032285:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.032288:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026115 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x749:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.032297:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520936.032298:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36226 00010000:00010000:0.0:1783520936.032304:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02611c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x74b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.032305:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.032306:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026115 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x749:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.032319:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.032325:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02611c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x74b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.032327:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026115 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x749:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.032328:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02610e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x74a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.032347:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02611c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x74b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.032353:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02610e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x74a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.032401:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02610e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x74a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.032404:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02611c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x74b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.032406:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350ade1c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+440:2387:x1870153002439808:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 334us (480us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.032408:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.032419:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02610e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x74a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.032425:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36225 00010000:00010000:0.0:1783520936.032432:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02611c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x74b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.032445:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520936.032449:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520936.032453:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.032456:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026115 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x749:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.032456:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.032459:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.032460:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02610e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x74a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.032462:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.032478:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.032480:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.032483:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026115 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x749:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.032486:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02610e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x74a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.032514:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026115 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x749:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.032516:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02610e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x74a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.032537:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02610e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x74a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.032542:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02610e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x74a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.032550:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516313b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+568:3627:x1870152993193856:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4533us (4597us total) trans 4294976135 rc 0/0 00010000:00010000:2.0:1783520936.032556:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.032560:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.032561:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44633 00000100:00100000:0.0:1783520936.032562:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516312680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+568:3626:x1870152993194240:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4249us (4326us total) trans 4294976136 rc 0/0 00010000:00010000:2.0:1783520936.032563:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.032570:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44634 00000100:00100000:1.0:1783520936.032605:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993195008:192.168.204.33@tcp:104: 00000004:00080000:0.0:1783520936.032612:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516311f80/2, rc 0, transno 4294976135 00010000:00010000:1.0:1783520936.032624:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026123 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x74c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.032626:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516311f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993193856:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.032628:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516311c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+566:3627:x1870152993193600:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4625us (4907us total) trans 4294976134 rc 0/0 00010000:00010000:3.0:1783520936.032644:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026115 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x749:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.032646:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516310a80/2, rc 0, transno 4294976134 00000100:00100000:2.0:1783520936.032647:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44632 00000100:00100000:0.0:1783520936.032656:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516310a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993193600:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.032668:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026115 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x749:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.032693:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.032734:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503326d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993196032:0@lo:6:osp-syn-1-0.0 00000004:00080000:0.0:1783520936.032736:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516310380/2, rc 0, transno 4294976136 00000100:00100000:0.0:1783520936.032749:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516310380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993194240:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.032752:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.032757:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.032757:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503327b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993195904:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.032765:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.032795:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.032802:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503324000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993196160:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.032825:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993196032 00000100:00100000:2.0:1783520936.032831:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993196032 00000100:00100000:0.0:1783520936.032841:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.032865:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993195904 00000100:00100000:2.0:1783520936.032869:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993196160 00000100:00100000:1.0:1783520936.032870:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993195904 00000100:00100000:2.0:1783520936.032876:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993196160 00000100:00100000:0.0:1783520936.032887:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44640 00000100:00100000:0.0:1783520936.032895:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+568:3628:x1870152993196032:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.032905:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1874 count 1 00000100:00100000:1.0:1783520936.032906:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44641 00000100:00100000:2.0:1783520936.032915:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44642 00000100:00100000:1.0:1783520936.032919:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+568:3627:x1870152993195904:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.032921:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026146 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x752:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.032930:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503325c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+568:3625:x1870152993196160:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.032934:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1873 count 1 00010000:00010000:0.0:1783520936.032942:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000000eca50ed/0xd5401cbabc020de0 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x752:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1842 expref: 439 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.032949:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02614d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x751:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.032950:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1875 count 1 00010000:00010000:0.0:1783520936.032951:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x752:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.032960:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x752:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.032966:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026154 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x753:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.032968:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026146 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x752:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.032971:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003152bea3/0xd5401cbabc020dd2 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x751:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1834 expref: 439 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.032993:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x751:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.032997:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x751:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.033003:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02614d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x751:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.033008:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000eeb776a3/0xd5401cbabc020de7 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x753:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1850 expref: 439 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.033010:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000000eca50ed/0xd5401cbabc020de0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x752:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1842 expref: 439 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.033013:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.033035:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x753:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.033040:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000000eca50ed/0xd5401cbabc020de0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x752:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1842 expref: 439 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.033041:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x753:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.033042:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003152bea3/0xd5401cbabc020dd2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x751:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1834 expref: 439 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.033052:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026154 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x753:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.033056:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.033074:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003152bea3/0xd5401cbabc020dd2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x751:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1834 expref: 439 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.033074:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503324380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993196288:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.033092:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000eeb776a3/0xd5401cbabc020de7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x753:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1850 expref: 439 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.033103:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a6300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993196416:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.033109:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000eeb776a3/0xd5401cbabc020de7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x753:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1850 expref: 439 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.033124:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ad8a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993196544:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.033129:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435122b6f00 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.033134:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3eb40 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.033146:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026131 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x74e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.033153:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026131 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x74e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.033159:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026131 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x74e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.033160:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895c00 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.033163:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993195520:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.033173:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026131 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x74e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.033182:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002439936 00010000:00010000:1.0:1783520936.033185:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026138 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x74f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.033187:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002439936 00010000:00010000:0.0:1783520936.033191:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.033196:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.033203:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026131 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x74e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.033209:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004d594477/0xd5401cbabc020dbd lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x74d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17fc expref: 440 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.033215:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.033221:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026131 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x74e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.033231:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026131 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x74e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.033237:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002440064 00000100:00100000:2.0:1783520936.033242:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002440064 00010000:00010000:3.0:1783520936.033245:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026131 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x74e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.033256:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000040113e79/0xd5401cbabc020db6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x74c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17ee expref: 441 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.033257:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026131 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x74e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.033285:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.033288:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.033290:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.033314:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36227 00000100:00100000:1.0:1783520936.033327:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350ade1f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+441:2387:x1870153002439936:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.033336:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36228 00010000:00010000:1.0:1783520936.033337:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.033345:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004d594477/0xd5401cbabc020dbd lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x74d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17fc expref: 441 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.033350:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350ade0700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+441:2390:x1870153002440064:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.033395:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004d594477/0xd5401cbabc020dbd lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x74d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17fc expref: 441 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.033396:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.033404:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000040113e79/0xd5401cbabc020db6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x74c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17ee expref: 441 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.033417:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503324700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+569:3627:x1870152993194880:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4256us (4364us total) trans 4294976137 rc 0/0 00010000:00010000:1.0:1783520936.033419:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004d594477/0xd5401cbabc020dbd lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x74d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17fc expref: 441 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.033420:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000040113e79/0xd5401cbabc020db6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x74c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17ee expref: 441 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.033431:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44637 00010000:00010000:1.0:1783520936.033453:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x74d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.033454:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000040113e79/0xd5401cbabc020db6 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x74c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f17ee expref: 440 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.033458:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x74d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.033467:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02612a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x74d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.033468:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350ade3480/2, rc 0, transno 4294976137 00010000:00010000:2.0:1783520936.033472:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x74c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.033480:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x74c:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.033483:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993194880:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.033502:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026123 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x74c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.033508:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02612a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x74d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.033531:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026123 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x74c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.033545:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.033557:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.033558:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363840f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993196672:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.033575:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.033581:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350ade1f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+439:2387:x1870153002439936:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 256us (574us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.033590:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350ade0700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+439:2390:x1870153002440064:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 243us (541us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.033594:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36227 00000100:00100000:0.0:1783520936.033601:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.033608:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36228 00000100:00100000:1.0:1783520936.033617:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002440192 00000100:00100000:1.0:1783520936.033622:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002440192 00000100:00100000:3.0:1783520936.033637:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.033638:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026123 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x74c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.033643:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000fa3c26b1/0xd5401cbabc020dcb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x74f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1818 expref: 438 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.033654:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026123 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x74c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.033660:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993196672 00010000:00010000:2.0:1783520936.033666:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026123 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x74c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.033666:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993196672 00000100:00100000:1.0:1783520936.033672:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002440320 00000100:00100000:1.0:1783520936.033677:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002440320 00010000:00010000:2.0:1783520936.033679:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026123 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x74c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.033698:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d1a24206/0xd5401cbabc020dd9 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x750:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1826 expref: 439 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.033702:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.033703:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44643 00010000:00010000:0.0:1783520936.033716:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:3.0:1783520936.033717:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36229 00010000:00010000:0.0:1783520936.033724:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026123 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x74c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.033732:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350ade1880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+439:2387:x1870153002440192:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.033741:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36230 00010000:00010000:3.0:1783520936.033748:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.033751:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba600 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520936.033758:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+439:2387:x1870153002440320:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.033760:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000fa3c26b1/0xd5401cbabc020dcb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x74f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1818 expref: 439 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.033760:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026123 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x74c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.033771:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.033781:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d1a24206/0xd5401cbabc020dd9 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x750:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1826 expref: 439 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.033782:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026123 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x74c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.033789:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635786050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+570:3626:x1870152993194624:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4908us (5077us total) trans 4294976138 rc 0/0 00000100:00100000:2.0:1783520936.033806:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44635 00010000:00010000:1.0:1783520936.033810:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d1a24206/0xd5401cbabc020dd9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x750:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1826 expref: 439 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.033813:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026123 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x74c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.033836:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026123 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x74c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.033845:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000fa3c26b1/0xd5401cbabc020dcb lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x74f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1818 expref: 439 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.033850:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x750:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.033856:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x750:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.033861:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02613f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x750:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.033863:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.033867:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02613f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x750:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.033868:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.033872:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000fa3c26b1/0xd5401cbabc020dcb lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x74f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1818 expref: 439 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.033873:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.033883:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.033888:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x74f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.033891:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x74f:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.033896:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363840ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+569:3626:x1870152993196672:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.033898:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026138 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x74f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.033906:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+438:2387:x1870153002440320:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 153us (271us total) trans 0 rc 0/0 00002000:00080000:0.0:1783520936.033912:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1876 count 1 00010000:00010000:3.0:1783520936.033915:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026138 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x74f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.033918:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36230 00010000:00010000:0.0:1783520936.033929:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02615b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x754:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.033943:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000004da1b51/0xd5401cbabc020dee lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x754:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f185e expref: 437 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.033944:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000004:00080000:2.0:1783520936.033944:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503325f80/2, rc 0, transno 4294976138 00010000:00010000:1.0:1783520936.033949:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026138 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x74f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.033960:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503325f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993194624:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.033964:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x754:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.033974:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x754:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.033975:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350ade1880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+437:2387:x1870153002440192:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 242us (397us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.033975:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026138 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x74f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.033982:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02615b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x754:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.033995:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36229 00010000:00010000:1.0:1783520936.034003:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026138 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x74f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.034011:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:0.0:1783520936.034022:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000004da1b51/0xd5401cbabc020dee lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x754:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f185e expref: 436 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.034023:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ad9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993196800:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.034027:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026138 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x74f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.034050:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000004da1b51/0xd5401cbabc020dee lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x754:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f185e expref: 436 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.034060:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.034062:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.034071:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363840f100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993196928:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.034080:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02612a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x74d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.034091:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.034092:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02612a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x74d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.034099:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026138 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x74f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.034106:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02612a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x74d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.034112:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.034119:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026138 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x74f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.034121:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c016cc0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.034133:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026138 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x74f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.034137:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993196800 00000100:00100000:0.0:1783520936.034143:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993196800 00010000:00010000:2.0:1783520936.034150:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026138 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x74f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.034153:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02612a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x74d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.034154:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503327800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+571:3626:x1870152993195392:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2704us (2845us total) trans 4294976139 rc 0/0 00000100:00100000:0.0:1783520936.034170:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44644 00010000:00010000:2.0:1783520936.034174:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026138 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x74f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.034182:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.034183:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511adbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+571:3628:x1870152993196800:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.034184:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44638 00010000:00010000:3.0:1783520936.034194:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.034200:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02612a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x74d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.034200:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1877 count 1 00010000:00010000:2.0:1783520936.034201:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.034206:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.034211:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.034213:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.034215:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026162 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x755:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.034219:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02612a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x74d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.034225:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503326300/2, rc 0, transno 4294976139 00010000:00010000:3.0:1783520936.034238:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02612a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x74d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.034239:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d964bed7/0xd5401cbabc020df5 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x755:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f186c expref: 436 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.034248:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503326300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993195392:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.034261:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x755:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.034265:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02612a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x74d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.034265:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x755:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.034272:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026162 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x755:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.034278:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ad9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993197056:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.034283:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc02612a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x74d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.034309:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.034318:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.034322:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.034324:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d964bed7/0xd5401cbabc020df5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x755:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f186c expref: 436 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.034326:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.034338:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d964bed7/0xd5401cbabc020df5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x755:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f186c expref: 436 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.034349:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363840ed80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993197184:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.034412:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350b83ef00 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.034432:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993197056 00000100:00100000:0.0:1783520936.034437:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993197056 00000100:00100000:1.0:1783520936.034463:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44645 00000100:00100000:1.0:1783520936.034471:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511adad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+572:3628:x1870152993197056:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.034481:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1878 count 1 00010000:00010000:1.0:1783520936.034490:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026169 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x756:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.034513:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000fcbad77a/0xd5401cbabc02291c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x756:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f300c expref: 436 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.034518:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x756:0x0].0x0 since=36s v=(0 0) 00010000:00010000:1.0:1783520936.034520:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x756:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.034522:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026169 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x756:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.034535:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000fcbad77a/0xd5401cbabc02291c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x756:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f300c expref: 436 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.034541:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000fcbad77a/0xd5401cbabc02291c lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x756:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f300c expref: 436 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.034548:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993197312:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.034560:0:35282:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d1a24206/0xd5401cbabc020dd9 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x750:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1826 expref: 436 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.034572:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.034576:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503326680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993195776:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.034596:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02613f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x750:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.034605:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02613f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x750:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.034613:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02613f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x750:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.034621:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02613f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x750:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.034629:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02613f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x750:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.034651:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000004:00080000:2.0:1783520936.034670:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503326a00/2, rc 0, transno 4294976140 00000100:00100000:3.0:1783520936.034671:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503325880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+572:3627:x1870152993194752:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5553us (5690us total) trans 4294976140 rc 0/0 00000100:00100000:3.0:1783520936.034683:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44636 00000100:00100000:2.0:1783520936.034685:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503326a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993194752:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.034743:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503327100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+573:3626:x1870152993195648:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2796us (2963us total) trans 4294976141 rc 0/0 00000100:00100000:0.0:1783520936.034750:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44639 00000004:00080000:0.0:1783520936.034787:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503327480/2, rc 0, transno 4294976141 00000100:00100000:0.0:1783520936.034798:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503327480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993195648:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.034804:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1783520936.034814:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511adb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993197440:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.034830:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.034837:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a6300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993196416:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.034840:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.034846:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02613f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x750:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.034851:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02614d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x751:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.034858:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea600 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520936.034861:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.034865:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02613f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x750:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.034876:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02613f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x750:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.034889:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02613f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x750:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.034900:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.034902:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02613f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x750:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.034926:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.034926:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:3.0:1783520936.034930:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.034933:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002440448 00010000:00010000:3.0:1783520936.034935:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.034937:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002440576 00000100:00100000:2.0:1783520936.034938:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002440448 00000100:00100000:0.0:1783520936.034939:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363840e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993197568:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.034943:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002440576 00000100:00100000:3.0:1783520936.034957:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993197440 00010000:00010000:2.0:1783520936.034961:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000000eca50ed/0xd5401cbabc020de0 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x752:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1842 expref: 437 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.034962:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993197440 00010000:00010000:1.0:1783520936.034966:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003152bea3/0xd5401cbabc020dd2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x751:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1834 expref: 437 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.034980:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.034992:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993197568 00000100:00100000:3.0:1783520936.034998:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993197568 00000100:00100000:2.0:1783520936.035003:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36231 00000100:00100000:1.0:1783520936.035011:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36232 00000100:00100000:2.0:1783520936.035016:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357d050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+437:2388:x1870153002440448:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.035022:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad8700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+437:2388:x1870153002440576:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520936.035025:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44646 00010000:00010000:2.0:1783520936.035026:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.035033:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.035035:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000000eca50ed/0xd5401cbabc020de0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x752:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1842 expref: 437 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.035037:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503324380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993196288:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.035038:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363840df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+573:3626:x1870152993197440:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.035041:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003152bea3/0xd5401cbabc020dd2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x751:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1834 expref: 437 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.035057:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1879 count 1 00010000:00010000:2.0:1783520936.035064:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000000eca50ed/0xd5401cbabc020de0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x752:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1842 expref: 437 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.035069:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026170 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x757:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.035072:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003152bea3/0xd5401cbabc020dd2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x751:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1834 expref: 437 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.035082:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026146 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x752:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.035090:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000eae88bd8/0xd5401cbabc020e03 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x757:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1888 expref: 437 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.035091:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000eca50ed/0xd5401cbabc020de0 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x752:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1842 expref: 437 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.035109:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003152bea3/0xd5401cbabc020dd2 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x751:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1834 expref: 437 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.035114:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x757:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.035121:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x757:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.035125:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x752:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.035125:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44647 00010000:00010000:3.0:1783520936.035127:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026170 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x757:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.035136:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x752:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.035139:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x751:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.035141:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363840c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+573:3625:x1870152993197568:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.035146:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026146 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x752:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.035148:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x751:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.035150:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000eae88bd8/0xd5401cbabc020e03 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x757:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1888 expref: 435 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.035159:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02614d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x751:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.035160:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1880 count 1 00010000:00010000:3.0:1783520936.035176:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000eae88bd8/0xd5401cbabc020e03 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x757:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1888 expref: 435 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.035176:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026146 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x752:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.035177:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026177 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x758:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.035178:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02614d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x751:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.035208:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351dc41500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993197696:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.035221:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.035231:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b9332861/0xd5401cbabc020e0a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x758:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1896 expref: 435 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.035239:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x758:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.035241:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.035244:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x758:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.035250:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026177 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x758:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.035254:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57d80 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.035260:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad8700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+435:2388:x1870153002440576:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 233us (356us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.035268:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36232 00000100:00100000:2.0:1783520936.035269:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357d050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+435:2388:x1870153002440448:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 255us (439us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.035278:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b9332861/0xd5401cbabc020e0a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x758:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1896 expref: 434 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.035283:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36231 00010000:00010000:1.0:1783520936.035291:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02614d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x751:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.035296:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b9332861/0xd5401cbabc020e0a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x758:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1896 expref: 434 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.035307:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02614d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x751:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.035317:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363840e680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993197824:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.035320:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02614d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x751:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.035329:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02614d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x751:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.035348:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.035353:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ad8a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993196544:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.035380:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.035387:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02614d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x751:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.035397:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.035401:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02614d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x751:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.035408:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026154 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x753:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.035411:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02614d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x751:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.035438:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02614d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x751:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.035446:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02614d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x751:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.035459:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+574:3627:x1870152993195904:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2542us (2663us total) trans 4294976142 rc 0/0 00010000:00010000:0.0:1783520936.035466:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.035468:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44641 00010000:00010000:0.0:1783520936.035470:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.035474:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:2.0:1783520936.035476:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503327b80/2, rc 0, transno 4294976142 00010000:00010000:0.0:1783520936.035487:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026146 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x752:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.035488:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503327b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993195904:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.035511:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026146 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x752:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.035520:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026146 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x752:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.035528:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026146 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x752:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.035545:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.035552:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.035556:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026146 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x752:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.035562:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.035564:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026146 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x752:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.035571:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026146 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x752:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.035580:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026146 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x752:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.035589:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026146 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x752:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.035603:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.035605:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.035607:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.035641:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+574:3628:x1870152993196032:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2746us (2879us total) trans 4294976143 rc 0/0 00000004:00080000:2.0:1783520936.035647:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503326d80/2, rc 0, transno 4294976143 00000100:00100000:0.0:1783520936.035651:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44640 00000100:00100000:2.0:1783520936.035654:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503326d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993196032:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.035690:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351dc41180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993197952:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.035691:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350b83e6c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.035714:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.035733:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1783520936.035738:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351dc41f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993198080:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.035741:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993197952 00000100:00100000:3.0:1783520936.035746:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993197952 00000100:00100000:2.0:1783520936.035754:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.035779:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993198080 00000100:00100000:3.0:1783520936.035783:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993198080 00000100:00100000:0.0:1783520936.035800:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44648 00000100:00100000:1.0:1783520936.035811:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44649 00000100:00100000:0.0:1783520936.035812:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+575:3627:x1870152993197952:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.035826:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.035827:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+575:3627:x1870152993198080:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.035832:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1881 count 1 00002000:00080000:1.0:1783520936.035843:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1882 count 1 00010000:00010000:0.0:1783520936.035847:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc02617e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x759:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.035851:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002440704 00000100:00100000:3.0:1783520936.035858:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002440704 00010000:00010000:1.0:1783520936.035858:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026185 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x75a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.035863:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.035869:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001ce0d25a/0xd5401cbabc020e11 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x759:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f18a4 expref: 434 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.035878:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d3318d51/0xd5401cbabc020e18 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x75a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f18b2 expref: 434 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.035881:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000eeb776a3/0xd5401cbabc020de7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x753:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1850 expref: 434 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.035881:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x759:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.035891:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x759:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.035901:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc02617e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x759:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.035902:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x75a:0x0].0x0 since=37s v=(0 0) 00000100:00100000:2.0:1783520936.035907:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002440832 00010000:00010000:1.0:1783520936.035911:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x75a:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.035914:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002440832 00010000:00010000:1.0:1783520936.035922:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026185 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x75a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.035923:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36233 00010000:00010000:2.0:1783520936.035937:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000004da1b51/0xd5401cbabc020dee lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x754:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f185e expref: 435 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.035939:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001ce0d25a/0xd5401cbabc020e11 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x759:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18a4 expref: 435 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.035949:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d3318d51/0xd5401cbabc020e18 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x75a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18b2 expref: 435 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.035968:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001ce0d25a/0xd5401cbabc020e11 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x759:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18a4 expref: 435 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.035971:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d3318d51/0xd5401cbabc020e18 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x75a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18b2 expref: 435 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.035978:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36234 00000100:00100000:2.0:1783520936.035991:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511adaa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+435:2388:x1870153002440832:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.035993:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363840c700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993198208:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.035998:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a6300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993198336:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.036003:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.036010:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000004da1b51/0xd5401cbabc020dee lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x754:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f185e expref: 435 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.036022:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000004da1b51/0xd5401cbabc020dee lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x754:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f185e expref: 435 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.036036:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x754:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.036040:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x754:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.036043:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3eb40 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.036043:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435122b6f00 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.036044:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02615b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x754:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.036055:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02615b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x754:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.036070:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.036075:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363840ed80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993197184:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.036093:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026162 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x755:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.036100:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+435:2388:x1870153002440704:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.036101:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511adaa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+435:2388:x1870153002440832:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 107us (236us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.036119:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.036121:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36234 00010000:00010000:3.0:1783520936.036128:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000eeb776a3/0xd5401cbabc020de7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x753:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1850 expref: 435 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.036148:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.036162:0:9151:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000004da1b51/0xd5401cbabc020dee lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x754:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f185e expref: 434 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.036177:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002440960 00000100:00100000:2.0:1783520936.036178:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363840f100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993196928:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.036180:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000eeb776a3/0xd5401cbabc020de7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x753:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1850 expref: 433 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.036183:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002440960 00000100:00100000:1.0:1783520936.036194:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.036200:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000eeb776a3/0xd5401cbabc020de7 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x753:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1850 expref: 434 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.036202:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02615b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x754:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.036207:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d964bed7/0xd5401cbabc020df5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x755:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f186c expref: 434 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.036214:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002441088 00010000:00010000:3.0:1783520936.036216:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x753:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.036220:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002441088 00010000:00010000:3.0:1783520936.036227:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x753:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.036231:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02615b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x754:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.036236:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026154 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x753:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.036243:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000fcbad77a/0xd5401cbabc02291c lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x756:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f300c expref: 434 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.036251:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36235 00010000:00010000:3.0:1783520936.036261:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026154 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x753:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.036267:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a6a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+434:2387:x1870153002440960:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.036268:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02615b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x754:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.036284:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.036292:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d964bed7/0xd5401cbabc020df5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x755:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f186c expref: 434 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.036294:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02615b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x754:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.036299:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.036307:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36236 00010000:00010000:2.0:1783520936.036317:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02615b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x754:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.036319:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d964bed7/0xd5401cbabc020df5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x755:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f186c expref: 434 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.036323:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+434:2388:x1870153002440704:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 387us (498us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.036324:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a4e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+434:2389:x1870153002441088:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.036341:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.036343:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36233 00010000:00010000:1.0:1783520936.036350:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000fcbad77a/0xd5401cbabc02291c lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x756:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f300c expref: 434 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.036356:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d964bed7/0xd5401cbabc020df5 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x755:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f186c expref: 433 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.036380:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.036385:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.036391:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02615b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x754:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.036395:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000fcbad77a/0xd5401cbabc02291c lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x756:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f300c expref: 433 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.036403:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.036410:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02615b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x754:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.036429:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02615b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x754:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.036429:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x755:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.036431:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x756:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.036439:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x755:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.036440:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x756:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.036450:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026162 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x755:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.036452:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026169 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x756:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.036456:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02615b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x754:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.036474:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363840ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+576:3626:x1870152993196672:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2758us (2875us total) trans 4294976144 rc 0/0 00010000:00010000:0.0:1783520936.036476:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026162 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x755:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.036479:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026169 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x756:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.036484:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02615b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x754:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.036508:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44643 00010000:00010000:0.0:1783520936.036540:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.036541:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.036545:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.036548:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:2.0:1783520936.036554:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363840f480/2, rc 0, transno 4294976144 00010000:00010000:1.0:1783520936.036557:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.036561:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a6a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+432:2387:x1870153002440960:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 298us (413us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.036567:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36235 00000100:00100000:2.0:1783520936.036574:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363840f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993196672:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.036588:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026162 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x755:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.036595:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026162 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x755:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.036598:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a4e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+431:2389:x1870153002441088:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 278us (403us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.036600:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026154 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x753:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.036610:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026162 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x755:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.036616:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36236 00010000:00010000:2.0:1783520936.036626:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026154 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x753:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.036628:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026162 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x755:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.036655:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026154 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x753:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.036655:0:35274:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000fcbad77a/0xd5401cbabc02291c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x756:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f300c expref: 430 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.036663:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351dc42680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993198464:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.036689:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993197312:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.036691:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026154 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x753:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.036698:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.036700:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026169 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x756:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.036711:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026169 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x756:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.036716:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026169 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x756:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.036721:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026169 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x756:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.036726:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026169 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x756:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.036728:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.036737:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993198464 00010000:00010000:1.0:1783520936.036742:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:0.0:1783520936.036744:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993198464 00010000:00010000:3.0:1783520936.036751:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520936.036756:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.036758:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026154 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x753:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.036763:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026169 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x756:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.036779:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.036781:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbbe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.036786:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026154 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x753:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.036788:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026169 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x756:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.036802:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503325c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+578:3625:x1870152993196160:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3875us (3959us total) trans 4294976145 rc 0/0 00010000:00010000:3.0:1783520936.036810:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026154 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x753:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.036816:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026169 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x756:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.036818:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511adad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+578:3628:x1870152993197056:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2347us (2508us total) trans 4294976146 rc 0/0 00000100:00100000:2.0:1783520936.036823:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44642 00000100:00100000:1.0:1783520936.036836:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44645 00010000:00010000:3.0:1783520936.036845:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026154 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x753:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.036849:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026169 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x756:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.036861:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026154 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x753:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.036864:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026169 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x756:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.036865:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943511ad9180/2, rc 0, transno 4294976146 00000100:00100000:1.0:1783520936.036880:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ad9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993197056:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.036895:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.036900:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.036900:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.036904:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.036904:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.036909:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.036929:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511adb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993198592:0@lo:6:osp-syn-1-0.0 00000004:00080000:0.0:1783520936.036937:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503324000/2, rc 0, transno 4294976145 00000100:00100000:0.0:1783520936.036953:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503324000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993196160:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.036956:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.037000:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993198592 00000100:00100000:0.0:1783520936.037005:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993198592 00000100:00100000:3.0:1783520936.037018:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44650 00000100:00100000:2.0:1783520936.037019:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351dc40380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993198720:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.037028:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363840ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+577:3625:x1870152993198464:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.037037:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44651 00002000:00080000:3.0:1783520936.037041:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1883 count 1 00000100:00100000:0.0:1783520936.037048:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+577:3626:x1870152993198592:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.037053:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00080000:0.0:1783520936.037062:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1884 count 1 00010000:00010000:0.0:1783520936.037073:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02618c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x75c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.037082:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993198720 00010000:00010000:0.0:1783520936.037082:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000fffc0d30/0xd5401cbabc020e26 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x75c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f18ce expref: 429 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.037087:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993198720 00010000:00010000:3.0:1783520936.037089:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026193 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x75b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.037109:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44652 00010000:00010000:0.0:1783520936.037109:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x75c:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.037115:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ace9688a/0xd5401cbabc020e1f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x75b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f18c0 expref: 429 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.037115:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x75c:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.037118:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+578:3627:x1870152993198720:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.037122:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02618c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x75c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.037137:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x75b:0x0].0x0 since=37s v=(0 0) 00002000:00080000:2.0:1783520936.037139:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1885 count 1 00010000:00010000:3.0:1783520936.037141:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x75b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.037147:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026193 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x75b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.037151:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000fffc0d30/0xd5401cbabc020e26 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x75c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18ce expref: 429 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.037158:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02619a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x75d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.037169:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000fffc0d30/0xd5401cbabc020e26 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x75c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18ce expref: 429 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.037173:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ace9688a/0xd5401cbabc020e1f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x75b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18c0 expref: 429 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.037180:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001409c586/0xd5401cbabc020e2d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x75d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f18dc expref: 429 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.037191:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363840d880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993198848:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.037200:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ace9688a/0xd5401cbabc020e1f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x75b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18c0 expref: 429 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.037205:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x75d:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.037210:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x75d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.037214:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02619a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x75d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.037226:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350b83ef00 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.037242:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001409c586/0xd5401cbabc020e2d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x75d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18dc expref: 429 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.037242:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.037255:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001409c586/0xd5401cbabc020e2d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x75d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18dc expref: 429 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.037264:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993199104:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.037274:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.037278:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026162 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x755:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.037291:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.037296:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026162 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x755:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.037302:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351dc40700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993198976:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.037307:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026162 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x755:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.037309:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511adbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+579:3628:x1870152993196800:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3130us (3250us total) trans 4294976147 rc 0/0 00000100:00100000:0.0:1783520936.037322:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44644 00010000:00010000:2.0:1783520936.037333:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026162 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x755:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.037337:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d83c0 going to sleep for 11 seconds 00000004:00080000:1.0:1783520936.037338:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943511ad9f80/2, rc 0, transno 4294976147 00010000:00010000:2.0:1783520936.037350:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026162 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x755:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.037355:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ad9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993196800:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.037418:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.037421:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.037425:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.037448:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363840f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993199232:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.037466:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363840e680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993197824:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.037481:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026177 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x758:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.037489:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351dc41500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993197696:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.037505:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.037524:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993199232 00010000:00010000:3.0:1783520936.037527:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026170 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x757:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.037528:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993199232 00000100:00100000:2.0:1783520936.037559:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44653 00000100:00100000:2.0:1783520936.037566:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+579:3628:x1870152993199232:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.037576:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1886 count 1 00010000:00010000:2.0:1783520936.037584:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0261a1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x75e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.037592:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.037599:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000045b20952/0xd5401cbabc022b76 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x75e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3266 expref: 429 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.037608:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x75e:0x0].0x0 since=36s v=(0 0) 00010000:00010000:2.0:1783520936.037612:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x75e:0x0].0x0 since=0s v=(1 0) 00000100:00100000:0.0:1783520936.037615:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002441216 00010000:00010000:2.0:1783520936.037617:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0261a1 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x75e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.037621:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002441216 00000100:00100000:3.0:1783520936.037631:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.037645:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000eae88bd8/0xd5401cbabc020e03 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x757:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1888 expref: 430 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.037649:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000045b20952/0xd5401cbabc022b76 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x75e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3266 expref: 430 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.037665:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002441344 00010000:00010000:2.0:1783520936.037670:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000045b20952/0xd5401cbabc022b76 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x75e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3266 expref: 430 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.037671:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002441344 00000100:00100000:2.0:1783520936.037686:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993199360:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.037689:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36237 00010000:00010000:1.0:1783520936.037693:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b9332861/0xd5401cbabc020e0a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x758:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1896 expref: 431 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.037704:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351dc41500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+431:2387:x1870153002441216:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.037716:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.037723:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000eae88bd8/0xd5401cbabc020e03 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x757:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1888 expref: 431 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.037729:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c016cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.037730:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36238 00010000:00010000:0.0:1783520936.037744:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000eae88bd8/0xd5401cbabc020e03 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x757:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1888 expref: 431 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.037745:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357e050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+431:2390:x1870153002441344:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.037750:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895c00 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.037762:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.037768:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000eae88bd8/0xd5401cbabc020e03 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x757:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1888 expref: 431 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.037770:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b9332861/0xd5401cbabc020e0a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x758:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1896 expref: 431 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.037786:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x757:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.037790:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b9332861/0xd5401cbabc020e0a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x758:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1896 expref: 430 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.037791:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x757:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.037800:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026170 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x757:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.037810:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026170 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x757:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.037813:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b9332861/0xd5401cbabc020e0a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x758:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1896 expref: 430 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.037831:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x758:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.037835:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x758:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.037838:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.037839:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026177 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x758:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.037849:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026170 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x757:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.037850:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026177 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x758:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.037862:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351dc41500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+429:2387:x1870153002441216:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 162us (303us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.037872:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026170 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x757:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.037877:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36237 00010000:00010000:3.0:1783520936.037888:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026170 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x757:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.037892:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.037898:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026170 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x757:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.037911:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026177 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x758:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.037916:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.037920:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357e050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+428:2390:x1870153002441344:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 176us (290us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.037924:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026177 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x758:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.037926:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.037932:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026170 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x757:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.037933:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36238 00010000:00010000:2.0:1783520936.037944:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026177 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x758:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.037945:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.037951:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026170 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x757:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.037954:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026177 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x758:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.037970:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026170 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x757:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.037986:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.037986:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026170 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x757:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.037990:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.037998:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026177 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x758:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.038000:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026170 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x757:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.038010:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbebe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.038016:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026177 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x758:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.038024:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026177 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x758:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.038031:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.038035:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026177 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x758:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.038035:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.038040:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.038042:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026177 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x758:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.038060:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.038062:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.038064:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.038083:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363840c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+581:3625:x1870152993197568:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2946us (3107us total) trans 4294976149 rc 0/0 00000100:00100000:2.0:1783520936.038089:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44647 00000004:00080000:0.0:1783520936.038096:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363840e300/2, rc 0, transno 4294976149 00000100:00100000:0.0:1783520936.038111:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363840e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993197568:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.038157:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993199488:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.038177:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.038203:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993199488 00000100:00100000:1.0:1783520936.038208:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993199488 00000100:00100000:1.0:1783520936.038235:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44654 00000100:00100000:1.0:1783520936.038244:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363840e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+581:3625:x1870152993199488:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.038259:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363840df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+581:3626:x1870152993197440:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3222us (3401us total) trans 4294976148 rc 0/0 00000100:00100000:3.0:1783520936.038268:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44646 00000004:00080000:2.0:1783520936.038269:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943511adb100/2, rc 0, transno 4294976148 00002000:00080000:1.0:1783520936.038270:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1887 count 1 00000100:00100000:2.0:1783520936.038282:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511adb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993197440:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.038285:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0261a8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x75f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.038297:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000258c9a5f/0xd5401cbabc020e34 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x75f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f18f8 expref: 427 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.038304:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x75f:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.038309:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x75f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.038313:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0261a8 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x75f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.038325:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503324380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993199616:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.038338:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000258c9a5f/0xd5401cbabc020e34 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x75f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18f8 expref: 427 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.038349:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.038349:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000258c9a5f/0xd5401cbabc020e34 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x75f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18f8 expref: 427 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.038386:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a4e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993199744:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.038397:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993199616 00000100:00100000:0.0:1783520936.038403:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993199616 00000100:00100000:1.0:1783520936.038424:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.038438:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44655 00000100:00100000:0.0:1783520936.038447:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350ade0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+581:3626:x1870152993199616:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.038460:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1888 count 1 00010000:00010000:0.0:1783520936.038473:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0261af lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x760:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.038485:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007982b26c/0xd5401cbabc020e42 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x760:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1906 expref: 427 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.038504:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x760:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.038508:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x760:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.038512:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0261af lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x760:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.038532:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007982b26c/0xd5401cbabc020e42 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x760:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1906 expref: 427 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.038542:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007982b26c/0xd5401cbabc020e42 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x760:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1906 expref: 427 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.038551:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503326680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993199872:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.038584:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435192190c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.039276:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.039312:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363840c700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993198208:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.039329:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002441472 00000100:00100000:3.0:1783520936.039334:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002441472 00010000:00010000:0.0:1783520936.039335:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc02617e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x759:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.039395:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a6300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993198336:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.039416:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026185 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x75a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.039419:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001ce0d25a/0xd5401cbabc020e11 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x759:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18a4 expref: 428 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.039467:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36239 00000100:00100000:0.0:1783520936.039478:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a4000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+428:2390:x1870153002441472:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.039486:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.039490:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.039510:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001ce0d25a/0xd5401cbabc020e11 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x759:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18a4 expref: 428 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.039528:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002441600 00010000:00010000:0.0:1783520936.039530:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001ce0d25a/0xd5401cbabc020e11 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x759:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18a4 expref: 428 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.039534:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002441600 00010000:00010000:0.0:1783520936.039553:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001ce0d25a/0xd5401cbabc020e11 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x759:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18a4 expref: 429 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.039556:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d3318d51/0xd5401cbabc020e18 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x75a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18b2 expref: 429 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.039572:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x759:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.039577:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x759:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.039583:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc02617e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x759:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.039594:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc02617e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x759:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.039599:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36240 00000100:00100000:2.0:1783520936.039610:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a5500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+428:2390:x1870153002441600:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.039619:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.039624:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d3318d51/0xd5401cbabc020e18 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x75a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18b2 expref: 428 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.039628:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.039639:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d3318d51/0xd5401cbabc020e18 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x75a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18b2 expref: 428 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.039656:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d3318d51/0xd5401cbabc020e18 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x75a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18b2 expref: 428 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.039661:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc02617e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x759:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.039663:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a4000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+428:2390:x1870153002441472:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 185us (384us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.039681:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x75a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.039683:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc02617e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x759:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.039683:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36239 00010000:00010000:2.0:1783520936.039689:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x75a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.039699:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026185 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.039703:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc02617e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x759:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.039717:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026185 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x75a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.039720:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc02617e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x759:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.039752:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.039755:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.039774:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a5500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+426:2390:x1870153002441600:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 166us (288us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.039782:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36240 00010000:00010000:2.0:1783520936.039802:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026185 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.039810:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026185 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.039819:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026185 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.039826:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026185 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x75a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.039841:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520936.039850:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520936.039856:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc02617e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x759:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.039864:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.039868:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc02617e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x759:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.039876:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.039885:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026185 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x75a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.039886:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc02617e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x759:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.039896:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+582:3627:x1870152993197952:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4085us (4177us total) trans 4294976150 rc 0/0 00010000:00010000:0.0:1783520936.039912:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.039917:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc02617e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x759:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.039919:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026185 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x75a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.039921:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44648 00010000:00010000:2.0:1783520936.039944:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc02617e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x759:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.039945:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026185 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x75a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.039966:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351dc41180/2, rc 0, transno 4294976150 00010000:00010000:0.0:1783520936.039972:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026185 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x75a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.039978:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351dc41180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993197952:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.039979:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.039982:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.039987:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.039989:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026185 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x75a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.040007:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.040009:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.040010:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.040042:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363840d880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993198848:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.040061:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02618c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x75c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.040062:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1783520936.040076:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363840c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993200000:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.040090:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+582:3627:x1870152993198080:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4268us (4335us total) trans 4294976151 rc 0/0 00000100:00100000:2.0:1783520936.040099:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44649 00000100:00100000:0.0:1783520936.040126:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.040142:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351dc40700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993198976:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.040160:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026193 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x75b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.040185:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993200000 00000100:00100000:2.0:1783520936.040190:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993200000 00000100:00100000:2.0:1783520936.040221:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44656 00000004:00080000:0.0:1783520936.040227:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351dc41f80/2, rc 0, transno 4294976151 00000100:00100000:2.0:1783520936.040232:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363840d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+582:3628:x1870152993200000:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.040238:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351dc41f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993198080:0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.040250:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1889 count 1 00010000:00010000:2.0:1783520936.040265:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0261b6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x761:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.040275:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:2.0:1783520936.040280:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000467a31ed/0xd5401cbabc020e49 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x761:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1914 expref: 425 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.040285:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993200128:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.040299:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x761:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.040302:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x761:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.040306:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0261b6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x761:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.040309:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.040333:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000467a31ed/0xd5401cbabc020e49 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x761:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1914 expref: 425 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.040342:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000467a31ed/0xd5401cbabc020e49 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x761:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1914 expref: 425 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.040353:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade1880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993200256:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.040422:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993200128 00000100:00100000:3.0:1783520936.040427:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993200128 00000100:00100000:2.0:1783520936.040455:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967a80 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.040467:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44657 00000100:00100000:3.0:1783520936.040476:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+583:3627:x1870152993200128:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520936.040488:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1890 count 1 00010000:00010000:3.0:1783520936.040516:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0261bd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x762:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.040529:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c3fbb250/0xd5401cbabc020e50 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x762:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1922 expref: 425 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.040536:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x762:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.040539:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x762:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.040542:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0261bd lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x762:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.040562:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c3fbb250/0xd5401cbabc020e50 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x762:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1922 expref: 425 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.040577:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c3fbb250/0xd5401cbabc020e50 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x762:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1922 expref: 425 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.040587:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351dc40700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993200384:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.040629:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f180 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.041319:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.041385:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002441728 00000100:00100000:0.0:1783520936.041393:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002441728 00000100:00100000:3.0:1783520936.041411:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.041417:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000fffc0d30/0xd5401cbabc020e26 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x75c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18ce expref: 426 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.041447:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002441856 00000100:00100000:0.0:1783520936.041451:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002441856 00000100:00100000:1.0:1783520936.041456:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36241 00010000:00010000:0.0:1783520936.041467:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ace9688a/0xd5401cbabc020e1f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x75b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18c0 expref: 427 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.041471:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351dc41500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+427:2387:x1870153002441728:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.041486:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.041504:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000fffc0d30/0xd5401cbabc020e26 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x75c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18ce expref: 427 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.041515:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36242 00010000:00010000:1.0:1783520936.041520:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000fffc0d30/0xd5401cbabc020e26 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x75c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18ce expref: 427 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.041528:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e471500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+427:2388:x1870153002441856:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.041540:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.041549:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000fffc0d30/0xd5401cbabc020e26 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x75c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18ce expref: 427 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.041549:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ace9688a/0xd5401cbabc020e1f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x75b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18c0 expref: 427 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.041551:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993199360:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.041567:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.041573:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0261a1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x75e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.041580:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ace9688a/0xd5401cbabc020e1f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x75b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18c0 expref: 427 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.041587:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x75c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.041592:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x75c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.041597:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ace9688a/0xd5401cbabc020e1f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x75b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18c0 expref: 426 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.041605:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02618c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.041610:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x75b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.041617:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x75b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.041627:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026193 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.041629:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02618c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x75c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.041648:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026193 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x75b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.041672:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.041674:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.041686:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026193 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.041696:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026193 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.041706:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026193 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.041707:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351dc41500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+425:2387:x1870153002441728:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 241us (388us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.041717:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e471500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+425:2388:x1870153002441856:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 187us (300us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.041727:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026193 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x75b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.041728:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36241 00000100:00100000:0.0:1783520936.041740:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36242 00010000:00010000:3.0:1783520936.041754:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.041754:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002441984 00000100:00100000:1.0:1783520936.041760:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002441984 00010000:00010000:1.0:1783520936.041781:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000045b20952/0xd5401cbabc022b76 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x75e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3266 expref: 424 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.041791:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503326680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993199872:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.041803:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.041810:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026193 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x75b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.041812:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0261af lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x760:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.041817:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade1c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993199104:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.041828:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36243 00000100:00100000:1.0:1783520936.041839:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e006d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+424:2387:x1870153002441984:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.041841:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02619a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x75d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.041850:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.041858:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000045b20952/0xd5401cbabc022b76 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x75e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3266 expref: 424 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.041870:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000045b20952/0xd5401cbabc022b76 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x75e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3266 expref: 424 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.041884:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000045b20952/0xd5401cbabc022b76 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x75e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3266 expref: 424 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.041895:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x75e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.041898:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x75e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.041903:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0261a1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.041911:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0261a1 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x75e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.041929:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.041933:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026193 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x75b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.041940:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.041943:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026193 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x75b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.041945:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.041963:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026193 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x75b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.041966:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0261a1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.041975:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e006d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+423:2387:x1870153002441984:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 136us (412us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.041978:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026193 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x75b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.041993:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36243 00010000:00010000:2.0:1783520936.042002:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0261a1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.042013:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002442112 00010000:00010000:3.0:1783520936.042019:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.042020:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002442112 00010000:00010000:2.0:1783520936.042022:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0261a1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.042024:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.042029:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.042036:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0261a1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x75e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.042040:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001409c586/0xd5401cbabc020e2d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x75d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18dc expref: 423 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.042049:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.042067:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.042073:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520936.042075:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002442240 00010000:00010000:3.0:1783520936.042079:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0261a1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x75e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.042080:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002442240 00000100:00100000:0.0:1783520936.042090:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.042092:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.042096:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0261a1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x75e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.042101:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000258c9a5f/0xd5401cbabc020e34 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x75f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18f8 expref: 424 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.042115:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002442368 00000100:00100000:0.0:1783520936.042119:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002442368 00000100:00100000:1.0:1783520936.042131:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36244 00010000:00010000:0.0:1783520936.042139:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007982b26c/0xd5401cbabc020e42 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x760:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1906 expref: 425 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.042144:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b6300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+425:2388:x1870153002442112:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.042159:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.042163:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0261a1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x75e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.042167:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001409c586/0xd5401cbabc020e2d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x75d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18dc expref: 425 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.042177:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36245 00010000:00010000:3.0:1783520936.042186:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0261a1 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x75e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.042192:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001409c586/0xd5401cbabc020e2d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x75d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18dc expref: 425 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.042193:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357fc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+425:2388:x1870153002442240:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.042205:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+584:3628:x1870152993199232:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4636us (4710us total) trans 4294976152 rc 0/0 00010000:00010000:3.0:1783520936.042207:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0261a1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x75e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.042209:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.042219:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000258c9a5f/0xd5401cbabc020e34 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x75f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18f8 expref: 425 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.042223:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001409c586/0xd5401cbabc020e2d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x75d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18dc expref: 425 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.042224:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44653 00010000:00010000:3.0:1783520936.042235:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.042239:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.042243:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.042250:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x75d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.042254:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x75d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.042257:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000258c9a5f/0xd5401cbabc020e34 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x75f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18f8 expref: 424 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.042260:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36246 00010000:00010000:1.0:1783520936.042261:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02619a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.042275:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b4700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+424:2390:x1870153002442368:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.042282:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02619a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x75d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.042289:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.042291:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x75f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.042296:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x75f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.042298:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007982b26c/0xd5401cbabc020e42 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x760:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1906 expref: 424 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.042301:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.042302:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0261a8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.042319:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0261a8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x75f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.042324:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007982b26c/0xd5401cbabc020e42 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x760:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1906 expref: 424 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.042331:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b6300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+424:2388:x1870153002442112:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 189us (386us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.042335:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363840ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+584:3625:x1870152993198464:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5306us (5638us total) trans 4294976153 rc 0/0 00010000:00010000:0.0:1783520936.042341:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.042348:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36244 00000100:00100000:3.0:1783520936.042353:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44650 00010000:00010000:2.0:1783520936.042380:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007982b26c/0xd5401cbabc020e42 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x760:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1906 expref: 423 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.042390:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357fc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+423:2388:x1870153002442240:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 201us (343us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.042397:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x760:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.042403:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36245 00010000:00010000:2.0:1783520936.042405:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x760:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.042411:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0261af lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x760:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.042418:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0261af lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x760:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.042427:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351dc42680/2, rc 0, transno 4294976153 00000004:00080000:1.0:1783520936.042439:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363840f100/2, rc 0, transno 4294976152 00010000:00010000:2.0:1783520936.042440:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.042442:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351dc42680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993198464:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.042458:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363840f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993199232:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.042466:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0261af lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x760:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.042468:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b4700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+421:2390:x1870153002442368:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 195us (380us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.042480:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36246 00010000:00010000:0.0:1783520936.042485:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0261af lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x760:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.042509:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0261af lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x760:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.042519:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0261af lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x760:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.042523:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02619a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.042526:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e473480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993200512:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.042541:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02619a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.042549:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.042552:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02619a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.042560:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02619a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x75d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.042564:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.042579:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.042585:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0261af lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x760:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.042593:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.042599:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0261af lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x760:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.042604:0:8410:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000258c9a5f/0xd5401cbabc020e34 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x75f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f18f8 expref: 420 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.042613:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0261af lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x760:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.042625:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a4e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993199744:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.042630:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0261af lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x760:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.042640:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0261af lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x760:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.042642:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:1.0:1783520936.042645:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0261a8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.042645:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350ade0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+584:3626:x1870152993199616:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4196us (4293us total) trans 4294976154 rc 0/0 00000100:00100000:3.0:1783520936.042656:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e473b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993200640:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.042667:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44655 00010000:00010000:1.0:1783520936.042672:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0261a8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.042676:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.042680:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.042684:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0261a8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.042686:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.042687:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993200512 00000100:00100000:0.0:1783520936.042692:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993200512 00000100:00100000:3.0:1783520936.042697:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.042707:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.042710:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0261a8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.042719:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0261a8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x75f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.042726:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.042734:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.042736:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02619a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x75d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.042745:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503324380/2, rc 0, transno 4294976154 00010000:00010000:0.0:1783520936.042746:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.042753:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02619a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x75d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.042759:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503324380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993199616:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.042767:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02619a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x75d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.042782:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02619a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x75d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.042783:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.042791:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0261a8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x75f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.042793:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02619a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x75d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.042810:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb600 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520936.042811:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+586:3627:x1870152993198720:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5691us (5757us total) trans 4294976155 rc 0/0 00010000:00010000:3.0:1783520936.042818:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0261a8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x75f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.042821:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44652 00010000:00010000:0.0:1783520936.042826:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.042831:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.042834:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363840e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+585:3625:x1870152993199488:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4589us (4651us total) trans 4294976156 rc 0/0 00010000:00010000:3.0:1783520936.042836:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0261a8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x75f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.042836:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.042851:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44654 00000004:00080000:2.0:1783520936.042852:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351dc40380/2, rc 0, transno 4294976155 00000100:00100000:0.0:1783520936.042853:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993200640 00010000:00010000:3.0:1783520936.042858:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0261a8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x75f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.042860:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993200640 00000100:00100000:2.0:1783520936.042861:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351dc40380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993198720:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.042873:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0261a8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x75f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.042878:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44658 00000100:00100000:1.0:1783520936.042890:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e471c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+585:3625:x1870152993200512:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.042897:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44659 00010000:00010000:3.0:1783520936.042900:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00002000:00080000:1.0:1783520936.042903:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1891 count 1 00010000:00010000:3.0:1783520936.042905:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.042910:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e472300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+585:3625:x1870152993200640:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.042911:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.042919:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0261c4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x763:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.042927:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1892 count 1 00010000:00010000:1.0:1783520936.042931:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003a660e7e/0xd5401cbabc020e57 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x763:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1930 expref: 419 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.042937:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x763:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.042941:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x763:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.042944:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0261c4 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x763:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.042945:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0261cb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x764:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.042965:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e470000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993200768:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.042966:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003a660e7e/0xd5401cbabc020e57 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x763:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1930 expref: 419 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.042968:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000019fce057/0xd5401cbabc020e5e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x764:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f193e expref: 419 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.042990:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x764:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.042992:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003a660e7e/0xd5401cbabc020e57 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x763:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1930 expref: 419 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.042995:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x764:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.043004:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0261cb lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x764:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.043006:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363d3a6a00/2, rc 0, transno 4294976156 00000100:00100000:0.0:1783520936.043008:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.043021:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a4e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993200896:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.043024:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993199488:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.043040:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000019fce057/0xd5401cbabc020e5e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x764:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f193e expref: 419 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.043041:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1783520936.043053:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e470380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993201024:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.043056:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000019fce057/0xd5401cbabc020e5e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x764:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f193e expref: 419 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.043068:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.043074:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e006d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993201152:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.043082:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e473800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993201280:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.043089:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.043110:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895c00 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.043125:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.043129:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993200768 00000100:00100000:0.0:1783520936.043135:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993200768 00000100:00100000:2.0:1783520936.043136:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993201024 00000100:00100000:2.0:1783520936.043142:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993201024 00000100:00100000:1.0:1783520936.043150:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993201280 00000100:00100000:1.0:1783520936.043155:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993201280 00000100:00100000:0.0:1783520936.043162:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44660 00000100:00100000:0.0:1783520936.043170:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+588:3627:x1870152993200768:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.043182:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44661 00002000:00080000:0.0:1783520936.043183:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1893 count 1 00000100:00100000:1.0:1783520936.043191:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44662 00000100:00100000:2.0:1783520936.043195:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+588:3627:x1870152993201024:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.043198:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0261d2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x765:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.043205:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e471f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+588:3625:x1870152993201280:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.043211:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1894 count 1 00010000:00010000:0.0:1783520936.043216:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a1785b44/0xd5401cbabc020e65 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x765:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f194c expref: 419 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.043223:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1895 count 1 00010000:00010000:0.0:1783520936.043225:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x765:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.043226:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0261d9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x766:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.043234:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x765:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.043241:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0261d2 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x765:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.043243:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000059f36943/0xd5401cbabc022a65 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x766:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3155 expref: 419 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.043243:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0261e0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x767:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.043266:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x766:0x0].0x0 since=36s v=(0 0) 00010000:00010000:2.0:1783520936.043270:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x766:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.043273:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0261d9 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x766:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.043276:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000025834151/0xd5401cbabc020e73 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x767:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1968 expref: 419 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.043281:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a1785b44/0xd5401cbabc020e65 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x765:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f194c expref: 419 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.043296:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000059f36943/0xd5401cbabc022a65 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x766:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3155 expref: 419 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.043297:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x767:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.043307:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x767:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.043308:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a1785b44/0xd5401cbabc020e65 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x765:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f194c expref: 419 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.043314:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000059f36943/0xd5401cbabc022a65 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x766:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3155 expref: 419 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.043317:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0261e0 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x767:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.043341:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b4a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993201408:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.043342:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993201536:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.043394:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000025834151/0xd5401cbabc020e73 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x767:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1968 expref: 419 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.043412:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000025834151/0xd5401cbabc020e73 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x767:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1968 expref: 419 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.043424:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a5180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993201664:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.043439:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff9436298a6480 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.043440:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435192190c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.043456:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.043481:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02618c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.043501:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02618c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.043512:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02618c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x75c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.043520:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02618c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x75c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.043540:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.043553:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade1880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993200256:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.043575:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0261b6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x761:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.043604:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.043633:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002442496 00000100:00100000:1.0:1783520936.043637:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+589:3626:x1870152993198592:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6591us (6679us total) trans 4294976157 rc 0/0 00000100:00100000:0.0:1783520936.043640:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002442496 00000100:00100000:1.0:1783520936.043649:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44651 00010000:00010000:0.0:1783520936.043663:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000467a31ed/0xd5401cbabc020e49 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x761:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1914 expref: 420 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.043690:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36247 00000100:00100000:0.0:1783520936.043699:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+420:2389:x1870153002442496:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.043707:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.043713:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000467a31ed/0xd5401cbabc020e49 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x761:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1914 expref: 420 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.043724:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000467a31ed/0xd5401cbabc020e49 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x761:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1914 expref: 420 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.043729:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.043736:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02618c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x75c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.043740:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000467a31ed/0xd5401cbabc020e49 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x761:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1914 expref: 420 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.043758:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.043761:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x761:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.043764:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02618c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x75c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.043766:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x761:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.043776:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.043776:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0261b6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x761:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.043780:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351dc40700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993200384:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.043786:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02618c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x75c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.043799:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0261bd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x762:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.043814:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02618c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x75c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.043819:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0261b6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x761:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.043827:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02618c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x75c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.043845:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.043846:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.043848:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.043853:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.043854:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0261b6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x761:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.043864:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0261b6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x761:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.043873:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0261b6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x761:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.043878:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943511adb480/2, rc 0, transno 4294976157 00000100:00100000:0.0:1783520936.043878:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+419:2389:x1870153002442496:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 179us (273us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.043887:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0261b6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x761:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.043893:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36247 00000100:00100000:3.0:1783520936.043896:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511adb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993198592:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.043914:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.043915:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002442624 00000100:00100000:0.0:1783520936.043920:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002442624 00010000:00010000:2.0:1783520936.043923:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.043928:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0261b6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x761:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.043934:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.043937:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c3fbb250/0xd5401cbabc020e50 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x762:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1922 expref: 419 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.043938:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0261b6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x761:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.043949:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e471500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993201792:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.043959:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0261b6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x761:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.043969:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0261b6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x761:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.043973:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0261b6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x761:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.043978:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36248 00000100:00100000:0.0:1783520936.043992:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a7b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+419:2390:x1870153002442624:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.043994:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.043997:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.044000:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.044003:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.044009:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c3fbb250/0xd5401cbabc020e50 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x762:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1922 expref: 419 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.044021:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c3fbb250/0xd5401cbabc020e50 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x762:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1922 expref: 419 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.044029:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.044043:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c3fbb250/0xd5401cbabc020e50 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x762:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1922 expref: 419 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.044059:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993201792 00010000:00010000:0.0:1783520936.044062:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x762:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.044065:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993201792 00010000:00010000:0.0:1783520936.044067:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x762:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.044073:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0261bd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x762:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.044076:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363840d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+590:3628:x1870152993200000:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3846us (3949us total) trans 4294976158 rc 0/0 00000004:00080000:3.0:1783520936.044080:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363840c700/2, rc 0, transno 4294976158 00000100:00100000:2.0:1783520936.044091:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44656 00010000:00010000:0.0:1783520936.044093:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0261bd lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x762:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.044094:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363840c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993200000:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.044103:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44663 00000100:00100000:1.0:1783520936.044117:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350ade1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+589:3626:x1870152993201792:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.044124:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.044132:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0261bd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x762:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.044135:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1896 count 1 00010000:00010000:2.0:1783520936.044143:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0261bd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x762:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.044149:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0261bd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x762:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.044153:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0261e7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x768:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.044155:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a7b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+418:2390:x1870153002442624:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 164us (381us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.044160:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0261bd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x762:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.044175:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36248 00010000:00010000:1.0:1783520936.044185:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000047daa4e3/0xd5401cbabc020e7a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x768:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1976 expref: 418 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.044194:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x768:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.044197:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.044198:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x768:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.044202:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.044205:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0261e7 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x768:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.044208:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0261bd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x762:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.044224:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.044228:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0261bd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x762:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.044239:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000047daa4e3/0xd5401cbabc020e7a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x768:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1976 expref: 417 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.044239:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0261bd lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x762:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.044266:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:1.0:1783520936.044266:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000047daa4e3/0xd5401cbabc020e7a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x768:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1976 expref: 417 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.044267:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0261bd lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x762:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.044277:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993201920:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.044280:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+590:3627:x1870152993200128:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3803us (3968us total) trans 4294976159 rc 0/0 00000100:00100000:2.0:1783520936.044299:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44657 00000100:00100000:1.0:1783520936.044300:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a4700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993202048:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.044304:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc0261bd lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x762:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.044313:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.044323:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993201920 00000100:00100000:2.0:1783520936.044329:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993201920 00010000:00010000:0.0:1783520936.044332:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.044337:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.044340:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.044342:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3eb40 going to sleep for 11 seconds 00000004:00080000:3.0:1783520936.044355:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b6a00/2, rc 0, transno 4294976159 00000100:00100000:2.0:1783520936.044390:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44664 00000100:00100000:2.0:1783520936.044399:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635786450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+590:3628:x1870152993201920:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.044413:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1897 count 1 00010000:00010000:2.0:1783520936.044428:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0261ee lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x769:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.044431:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993200128:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.044443:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000012f9aaef/0xd5401cbabc020e81 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x769:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1984 expref: 417 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.044445:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993202176:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.044458:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x769:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.044461:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x769:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.044466:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0261ee lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x769:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.044484:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.044503:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000012f9aaef/0xd5401cbabc020e81 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x769:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1984 expref: 417 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.044516:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000012f9aaef/0xd5401cbabc020e81 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x769:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1984 expref: 417 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.044528:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993202304:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.044532:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a4e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993200896:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.044543:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993202176 00000100:00100000:1.0:1783520936.044549:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993202176 00010000:00010000:0.0:1783520936.044554:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0261c4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x763:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.044578:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f180 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.044584:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44665 00000100:00100000:1.0:1783520936.044592:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+591:3628:x1870152993202176:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.044605:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1898 count 1 00000100:00100000:2.0:1783520936.044609:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e006d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993201152:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.044620:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0261f5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x76a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.044625:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0261cb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x764:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.044640:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000026ec51ec/0xd5401cbabc020e88 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x76a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1992 expref: 417 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.044650:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x76a:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.044653:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x76a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.044658:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0261f5 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x76a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.044685:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000026ec51ec/0xd5401cbabc020e88 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x76a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1992 expref: 417 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.044695:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000026ec51ec/0xd5401cbabc020e88 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x76a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1992 expref: 417 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.044706:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a5500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993202432:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.044749:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.044888:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.044914:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993201536:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.044933:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002442752 00010000:00010000:0.0:1783520936.044934:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0261d9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x766:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.044940:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002442752 00010000:00010000:1.0:1783520936.044961:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000019fce057/0xd5401cbabc020e5e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x764:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f193e expref: 418 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.044995:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36249 00000100:00100000:1.0:1783520936.045005:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e006d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+418:2388:x1870153002442752:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.045013:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.045018:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000019fce057/0xd5401cbabc020e5e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x764:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f193e expref: 418 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.045028:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000019fce057/0xd5401cbabc020e5e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x764:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f193e expref: 418 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.045042:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000019fce057/0xd5401cbabc020e5e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x764:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f193e expref: 418 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.045053:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x764:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.045056:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x764:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.045060:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0261cb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x764:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.045068:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0261cb lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x764:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.045088:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.045102:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0261cb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x764:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.045112:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0261cb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x764:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.045122:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e006d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+417:2388:x1870153002442752:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 117us (234us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.045123:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0261cb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x764:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.045137:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0261cb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x764:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.045138:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36249 00010000:00010000:2.0:1783520936.045155:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.045223:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e472300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+592:3625:x1870152993200640:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2319us (2530us total) trans 4294976160 rc 0/0 00000100:00100000:2.0:1783520936.045228:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44659 00000004:00080000:0.0:1783520936.045246:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e473b80/2, rc 0, transno 4294976160 00000100:00100000:0.0:1783520936.045262:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e473b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993200640:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.045298:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.045306:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0261cb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x764:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.045307:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.045316:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993202560:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.045320:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.045325:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0261cb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x764:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.045334:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0261cb lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x764:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.045353:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.045405:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002442880 00010000:00010000:3.0:1783520936.045410:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0261cb lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x764:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.045410:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002442880 00000100:00100000:1.0:1783520936.045414:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b4a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993201408:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.045426:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0261cb lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x764:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.045431:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000059f36943/0xd5401cbabc022a65 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x766:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3155 expref: 417 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.045436:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0261d2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x765:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.045460:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.045464:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.045468:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.045471:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36250 00000100:00100000:2.0:1783520936.045481:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509a80380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+417:2390:x1870153002442880:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520936.045484:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993202560 00000100:00100000:3.0:1783520936.045489:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993202560 00010000:00010000:2.0:1783520936.045502:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.045509:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000059f36943/0xd5401cbabc022a65 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x766:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3155 expref: 417 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.045526:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000059f36943/0xd5401cbabc022a65 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x766:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3155 expref: 417 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.045538:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44666 00010000:00010000:2.0:1783520936.045543:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000059f36943/0xd5401cbabc022a65 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x766:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3155 expref: 417 pid: 35276 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.045552:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+592:3625:x1870152993202560:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.045558:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x766:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.045562:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x766:0x0].0x0 since=0s v=(1 0) 00002000:00080000:3.0:1783520936.045567:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1899 count 1 00010000:00010000:2.0:1783520936.045567:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0261d9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x766:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.045579:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0261d9 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x766:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.045599:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.045611:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0261d9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x766:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.045622:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0261d9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x766:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.045629:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509a80380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+416:2390:x1870153002442880:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 148us (324us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.045645:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36250 00010000:00010000:0.0:1783520936.045648:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0261d9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x766:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.045658:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0261d9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x766:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.045663:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0261fc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x76b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.045676:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.045680:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007595d9cd/0xd5401cbabc020e8f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x76b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f19a0 expref: 415 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.045685:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.045692:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x76b:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.045693:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0261d9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x766:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.045700:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x76b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.045710:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0261fc lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x76b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.045710:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.045717:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0261d9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x766:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.045734:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0261d9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x766:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.045739:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007595d9cd/0xd5401cbabc020e8f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x76b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19a0 expref: 415 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.045750:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.045754:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0261d9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x766:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.045763:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0261d9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x766:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.045769:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+593:3627:x1870152993201024:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2576us (2678us total) trans 4294976161 rc 0/0 00000100:00100000:0.0:1783520936.045779:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44661 00010000:00010000:2.0:1783520936.045788:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.045793:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.045796:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.045798:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.045807:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007595d9cd/0xd5401cbabc020e8f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x76b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19a0 expref: 415 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.045819:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993202688:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.045823:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002443008 00000100:00100000:1.0:1783520936.045827:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002443136 00000100:00100000:0.0:1783520936.045829:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002443008 00000100:00100000:1.0:1783520936.045833:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002443136 00000004:00080000:3.0:1783520936.045845:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e470380/2, rc 0, transno 4294976161 00010000:00010000:0.0:1783520936.045849:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a1785b44/0xd5401cbabc020e65 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x765:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f194c expref: 417 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.045856:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000025834151/0xd5401cbabc020e73 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x767:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1968 expref: 417 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.045859:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e470380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993201024:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.045883:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36251 00000100:00100000:0.0:1783520936.045895:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a4e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+417:2387:x1870153002443008:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.045899:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36252 00010000:00010000:0.0:1783520936.045906:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.045914:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357c450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+417:2390:x1870153002443136:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.045914:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a1785b44/0xd5401cbabc020e65 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x765:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f194c expref: 417 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.045916:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943634571c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993202816:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.045930:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.045934:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a1785b44/0xd5401cbabc020e65 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x765:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f194c expref: 417 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.045939:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000025834151/0xd5401cbabc020e73 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x767:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1968 expref: 417 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.045953:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.045966:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000025834151/0xd5401cbabc020e73 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x767:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1968 expref: 417 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.045971:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a1785b44/0xd5401cbabc020e65 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x765:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f194c expref: 417 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.045987:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x765:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.045988:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x767:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.045992:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x765:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.045994:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x767:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.045997:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0261d2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x765:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.046002:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0261e0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x767:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.046008:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0261d2 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x765:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.046016:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0261e0 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x767:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.046027:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993202816 00010000:00010000:0.0:1783520936.046032:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.046033:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993202816 00010000:00010000:1.0:1783520936.046036:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.046063:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a4e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+416:2387:x1870153002443008:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 172us (315us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.046066:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357c450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+416:2390:x1870153002443136:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 156us (274us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.046073:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44667 00000100:00100000:0.0:1783520936.046080:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36251 00000100:00100000:2.0:1783520936.046088:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634573b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+593:3626:x1870152993202816:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.046089:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36252 00002000:00080000:2.0:1783520936.046123:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1900 count 1 00010000:00010000:2.0:1783520936.046137:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026203 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x76c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.046139:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0261d2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x765:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.046150:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c016cc0 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.046160:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0261d2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x765:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.046161:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000713a27b6/0xd5401cbabc020e96 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x76c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f19ae expref: 414 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.046184:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x76c:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.046185:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0261d2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x765:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.046190:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x76c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.046199:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026203 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x76c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.046202:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0261d2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x765:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.046211:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a4700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993202048:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.046229:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0261e7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x768:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.046237:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.046238:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000713a27b6/0xd5401cbabc020e96 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x76c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19ae expref: 414 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.046251:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000713a27b6/0xd5401cbabc020e96 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x76c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19ae expref: 414 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.046263:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a83100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993202944:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.046276:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.046281:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0261d2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x765:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.046288:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.046293:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0261d2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x765:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.046301:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0261d2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x765:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.046313:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0261d2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x765:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.046320:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0261d2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x765:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.046328:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+594:3627:x1870152993200768:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3156us (3323us total) trans 4294976162 rc 0/0 00000004:00080000:3.0:1783520936.046338:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e470000/2, rc 0, transno 4294976162 00000100:00100000:1.0:1783520936.046348:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44660 00010000:00010000:2.0:1783520936.046355:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.046395:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.046397:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.046412:0:35282:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000025834151/0xd5401cbabc020e73 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x767:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1968 expref: 414 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.046427:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a5180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993201664:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.046443:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0261e0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x767:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.046457:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895c00 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.046460:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1783520936.046478:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993203072:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.046505:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993202304:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.046519:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.046526:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0261ee lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x769:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.046551:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993203072 00000100:00100000:2.0:1783520936.046555:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993203072 00010000:00010000:1.0:1783520936.046555:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0261e0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x767:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.046558:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e470000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993200768:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.046571:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0261e0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x767:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.046580:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0261e0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x767:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.046587:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44668 00010000:00010000:1.0:1783520936.046589:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0261e0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x767:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.046600:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+594:3628:x1870152993203072:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.046606:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00002000:00080000:2.0:1783520936.046613:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1901 count 1 00010000:00010000:0.0:1783520936.046614:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.046622:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0261e0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x767:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.046629:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02620a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x76d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.046631:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.046638:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0261e0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x767:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.046647:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002251ab7d/0xd5401cbabc020ea4 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x76d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f19bc expref: 413 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.046651:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0261e0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x767:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.046661:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x76d:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.046668:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x76d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.046674:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02620a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x76d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.046675:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0261e0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x767:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.046684:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e471f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+595:3625:x1870152993201280:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3480us (3556us total) trans 4294976163 rc 0/0 00010000:00010000:0.0:1783520936.046699:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0261e0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x767:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.046700:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44662 00010000:00010000:2.0:1783520936.046707:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002251ab7d/0xd5401cbabc020ea4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x76d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19bc expref: 413 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.046722:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002251ab7d/0xd5401cbabc020ea4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x76d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19bc expref: 413 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.046725:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.046729:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.046733:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.046736:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a80380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993203200:192.168.204.33@tcp:104: 00000004:00080000:3.0:1783520936.046738:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e473800/2, rc 0, transno 4294976163 00000100:00100000:3.0:1783520936.046753:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e473800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993201280:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.046776:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a5500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993202432:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.046779:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f180 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.046791:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0261f5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x76a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.046809:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993203328:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.046826:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.046851:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993203328 00000100:00100000:0.0:1783520936.046857:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993203328 00000100:00100000:0.0:1783520936.046882:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44669 00000100:00100000:0.0:1783520936.046890:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+595:3628:x1870152993203328:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.046901:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1902 count 1 00010000:00010000:0.0:1783520936.046913:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026211 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x76e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.046925:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000022f7e7e4/0xd5401cbabc022954 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x76e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3044 expref: 413 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.046931:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x76e:0x0].0x0 since=36s v=(0 0) 00010000:00010000:0.0:1783520936.046935:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x76e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.046938:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026211 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x76e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.046956:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000022f7e7e4/0xd5401cbabc022954 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x76e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3044 expref: 413 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.046965:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000022f7e7e4/0xd5401cbabc022954 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x76e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3044 expref: 413 pid: 8410 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.046974:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b5180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993203456:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.047005:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350b83e6c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.047295:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.047312:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.047329:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.047341:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002443264 00000100:00100000:1.0:1783520936.047347:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002443264 00000100:00100000:2.0:1783520936.047351:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002443520 00010000:00010000:1.0:1783520936.047397:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000047daa4e3/0xd5401cbabc020e7a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x768:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1976 expref: 414 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.047407:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002443520 00000100:00100000:3.0:1783520936.047417:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002443392 00000100:00100000:3.0:1783520936.047423:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002443392 00000100:00100000:1.0:1783520936.047425:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36253 00010000:00010000:2.0:1783520936.047431:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000026ec51ec/0xd5401cbabc020e88 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x76a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1992 expref: 416 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.047438:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b5500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+416:2388:x1870153002443264:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.047447:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000012f9aaef/0xd5401cbabc020e81 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x769:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1984 expref: 416 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.047455:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.047463:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000047daa4e3/0xd5401cbabc020e7a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x768:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1976 expref: 416 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.047470:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36254 00010000:00010000:1.0:1783520936.047483:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000047daa4e3/0xd5401cbabc020e7a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x768:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1976 expref: 416 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.047486:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b4000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+416:2388:x1870153002443520:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.047486:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36255 00010000:00010000:2.0:1783520936.047512:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.047513:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b4700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+416:2388:x1870153002443392:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.047520:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000026ec51ec/0xd5401cbabc020e88 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x76a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1992 expref: 416 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.047522:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000047daa4e3/0xd5401cbabc020e7a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x768:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1976 expref: 416 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.047525:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.047535:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000012f9aaef/0xd5401cbabc020e81 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x769:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1984 expref: 416 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.047547:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000026ec51ec/0xd5401cbabc020e88 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x76a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1992 expref: 416 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.047550:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x768:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.047557:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000012f9aaef/0xd5401cbabc020e81 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x769:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1984 expref: 415 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.047559:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x768:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.047569:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0261e7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x768:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.047577:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000026ec51ec/0xd5401cbabc020e88 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x76a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1992 expref: 415 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.047591:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000012f9aaef/0xd5401cbabc020e81 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x769:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1984 expref: 415 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.047597:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0261e7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x768:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.047604:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x76a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.047613:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x76a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.047624:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0261f5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.047630:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x769:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.047633:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0261f5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x76a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.047635:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x769:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.047639:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.047646:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0261ee lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x769:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.047657:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.047660:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0261ee lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x769:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.047664:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.047672:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b5500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+413:2388:x1870153002443264:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 237us (374us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.047681:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36253 00010000:00010000:0.0:1783520936.047684:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.047693:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b4000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+412:2388:x1870153002443520:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 210us (375us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.047700:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002443648 00000100:00100000:2.0:1783520936.047701:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36254 00000100:00100000:1.0:1783520936.047707:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002443648 00000100:00100000:0.0:1783520936.047709:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b4700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+412:2388:x1870153002443392:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 201us (377us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.047716:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36255 00010000:00010000:1.0:1783520936.047727:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003a660e7e/0xd5401cbabc020e57 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x763:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1930 expref: 412 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.047731:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0261ee lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x769:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.047747:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0261ee lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x769:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.047755:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0261ee lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x769:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.047765:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0261ee lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x769:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.047766:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36256 00000100:00100000:1.0:1783520936.047780:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515776680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+411:2390:x1870153002443648:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.047790:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.047791:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.047795:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.047800:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003a660e7e/0xd5401cbabc020e57 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x763:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1930 expref: 411 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.047802:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0261ee lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x769:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.047817:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.047817:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003a660e7e/0xd5401cbabc020e57 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x763:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1930 expref: 411 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.047822:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0261ee lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x769:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.047837:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0261ee lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x769:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.047843:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003a660e7e/0xd5401cbabc020e57 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x763:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1930 expref: 411 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.047852:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0261ee lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x769:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.047869:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0261ee lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x769:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.047869:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x763:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.047878:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x763:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.047884:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635786450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+596:3628:x1870152993201920:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3484us (3571us total) trans 4294976164 rc 0/0 00010000:00010000:1.0:1783520936.047890:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0261c4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x763:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.047902:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44664 00010000:00010000:3.0:1783520936.047908:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.047910:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0261c4 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x763:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.047912:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.047916:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.047938:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000004:00080000:2.0:1783520936.047940:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350ade1c00/2, rc 0, transno 4294976164 00010000:00010000:0.0:1783520936.047945:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0261c4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x763:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.047946:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993202688:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.047954:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993201920:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.047963:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0261c4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x763:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.047970:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0261fc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x76b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.047971:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515776680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+410:2390:x1870153002443648:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 190us (316us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.047972:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0261c4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x763:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.047991:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36256 00010000:00010000:0.0:1783520936.047993:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0261c4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x763:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.048016:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:0.0:1783520936.048017:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.048029:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0261e7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x768:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.048030:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.048037:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0261c4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x763:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.048043:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0261e7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x768:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.048051:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.048057:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0261c4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x763:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.048059:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0261e7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x768:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.048072:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0261c4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x763:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.048075:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0261e7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x768:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.048095:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e471c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+596:3625:x1870152993200512:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5206us (5530us total) trans 4294976165 rc 0/0 00010000:00010000:2.0:1783520936.048096:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0261c4 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x763:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.048104:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.048112:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44658 00010000:00010000:2.0:1783520936.048115:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0261c4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x763:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.048130:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.048134:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0261e7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x768:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.048141:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.048144:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0261e7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x768:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.048157:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0261e7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x768:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.048167:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0261e7 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x768:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.048174:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0261e7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x768:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.048184:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350ade1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+596:3626:x1870152993201792:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4069us (4154us total) trans 4294976166 rc 0/0 00010000:00010000:2.0:1783520936.048188:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.048191:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.048195:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.048195:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44663 00000100:00100000:2.0:1783520936.048222:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517330000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993203584:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.048249:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.048258:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.048266:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.048299:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002443776 00000004:00080000:2.0:1783520936.048303:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e471500/2, rc 0, transno 4294976166 00000100:00100000:0.0:1783520936.048304:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.048306:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002443776 00000100:00100000:2.0:1783520936.048318:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e471500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993201792:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.048327:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993203584 00000100:00100000:1.0:1783520936.048332:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993203584 00010000:00010000:3.0:1783520936.048396:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007595d9cd/0xd5401cbabc020e8f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x76b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19a0 expref: 410 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.048401:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44670 00000004:00080000:2.0:1783520936.048402:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e473480/2, rc 0, transno 4294976165 00000100:00100000:2.0:1783520936.048417:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e473480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993200512:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.048417:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517332300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+596:3626:x1870152993203584:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.048427:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002443904 00000100:00100000:2.0:1783520936.048428:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1783520936.048433:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002443904 00002000:00080000:1.0:1783520936.048441:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1903 count 1 00010000:00010000:3.0:1783520936.048452:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000713a27b6/0xd5401cbabc020e96 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x76c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19ae expref: 411 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.048458:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026218 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x76f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.048476:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000019519042/0xd5401cbabc020eab lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x76f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f19d8 expref: 411 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.048483:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x76f:0x0].0x0 since=37s v=(0 0) 00000100:00100000:3.0:1783520936.048485:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36257 00000100:00100000:2.0:1783520936.048488:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:1.0:1783520936.048489:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x76f:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.048504:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36258 00010000:00010000:1.0:1783520936.048508:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026218 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x76f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.048512:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517330700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993203840:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.048513:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b4a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+411:2388:x1870153002443776:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.048519:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b6680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+411:2389:x1870153002443904:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.048532:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.048534:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.048542:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007595d9cd/0xd5401cbabc020e8f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x76b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19a0 expref: 411 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.048543:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000713a27b6/0xd5401cbabc020e96 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x76c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19ae expref: 411 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.048553:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.048554:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000019519042/0xd5401cbabc020eab lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x76f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19d8 expref: 411 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.048566:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000713a27b6/0xd5401cbabc020e96 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x76c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19ae expref: 411 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.048577:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007595d9cd/0xd5401cbabc020e8f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x76b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19a0 expref: 411 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.048582:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000019519042/0xd5401cbabc020eab lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x76f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19d8 expref: 411 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.048594:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517330a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993203712:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.048597:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x76c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.048606:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007595d9cd/0xd5401cbabc020e8f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x76b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19a0 expref: 411 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.048606:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x76c:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.048610:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993203968:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.048618:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026203 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.048627:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.048634:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x76b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.048637:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026203 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x76c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.048639:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x76b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.048649:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0261fc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.048658:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.048660:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0261fc lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x76b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.048677:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a80380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993203200:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.048683:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c3696c0 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.048684:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.048685:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b6680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+410:2389:x1870153002443904:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 171us (380us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.048696:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36258 00010000:00010000:2.0:1783520936.048698:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02620a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x76d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.048709:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993203840 00000100:00100000:3.0:1783520936.048714:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b4a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+409:2388:x1870153002443776:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 204us (449us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.048715:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993203840 00000100:00100000:3.0:1783520936.048725:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36257 00010000:00010000:2.0:1783520936.048735:0:20415:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000713a27b6/0xd5401cbabc020e96 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x76c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19ae expref: 409 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.048749:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a83100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993202944:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.048750:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993203712 00000100:00100000:1.0:1783520936.048756:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993203712 00010000:00010000:2.0:1783520936.048768:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026203 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.048779:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026203 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.048787:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44671 00010000:00010000:2.0:1783520936.048788:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026203 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.048800:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517333100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+598:3628:x1870152993203840:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.048801:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026203 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.048811:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026203 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x76c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.048813:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1905 count 1 00000100:00100000:0.0:1783520936.048814:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44672 00010000:00010000:3.0:1783520936.048826:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0261fc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.048831:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517330380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+598:3625:x1870152993203712:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.048835:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02621f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x771:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.048840:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.048847:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0261fc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.048851:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.048856:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000043dbd9a7/0xd5401cbabc020eb9 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x771:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f19f4 expref: 407 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.048859:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026203 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x76c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.048878:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x771:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.048880:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.048884:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x771:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.048886:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026203 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x76c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.048890:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02621f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x771:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.048901:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026203 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x76c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.048912:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026203 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x76c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.048917:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026203 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x76c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.048922:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000043dbd9a7/0xd5401cbabc020eb9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x771:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19f4 expref: 407 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.048934:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000043dbd9a7/0xd5401cbabc020eb9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x771:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19f4 expref: 407 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.048935:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.048938:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.048940:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634573b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+599:3626:x1870152993202816:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2854us (2987us total) trans 4294976167 rc 0/0 00010000:00010000:0.0:1783520936.048940:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00002000:00080000:0.0:1783520936.048953:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1904 count 1 00000100:00100000:1.0:1783520936.048960:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993204096:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.048966:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44667 00010000:00010000:0.0:1783520936.048966:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026226 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x770:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.048972:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0261fc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.048987:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e8a94b3e/0xd5401cbabc020eb2 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x770:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f19e6 expref: 407 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.048990:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0261fc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x76b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.048997:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x770:0x0].0x0 since=37s v=(0 0) 00000004:00080000:2.0:1783520936.049000:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943634571c00/2, rc 0, transno 4294976167 00010000:00010000:0.0:1783520936.049005:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x770:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.049008:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435192190c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.049010:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026226 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x770:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.049013:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943634571c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993202816:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.049032:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e8a94b3e/0xd5401cbabc020eb2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x770:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19e6 expref: 407 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.049036:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0261f5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.049042:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e8a94b3e/0xd5401cbabc020eb2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x770:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19e6 expref: 407 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.049044:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.049054:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0261f5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.049057:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517333800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993204352:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.049059:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520936.049060:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b5880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993204224:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.049068:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0261fc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x76b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.049076:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0261f5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.049079:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.049087:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0261fc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x76b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.049088:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.049097:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0261f5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x76a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.049099:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350b83ef00 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.049103:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0261fc lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x76b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.049120:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993204352 00000100:00100000:2.0:1783520936.049124:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993204352 00010000:00010000:3.0:1783520936.049125:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0261fc lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x76b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.049133:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.049135:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc0261fc lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x76b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.049136:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.049141:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0261f5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x76a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.049156:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.049159:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0261f5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x76a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.049166:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0261f5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x76a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.049169:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.049174:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.049178:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0261f5 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x76a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.049180:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.049186:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0261f5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x76a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.049195:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.049212:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.049216:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.049219:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.049230:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+601:3628:x1870152993202176:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4636us (4744us total) trans 4294976168 rc 0/0 00000100:00100000:1.0:1783520936.049239:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44665 00000004:00080000:2.0:1783520936.049245:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b5c00/2, rc 0, transno 4294976168 00000100:00100000:2.0:1783520936.049258:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993202176:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.049279:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.049286:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+600:3625:x1870152993202560:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3737us (3932us total) trans 4294976169 rc 0/0 00000100:00100000:3.0:1783520936.049296:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44666 00000100:00100000:2.0:1783520936.049302:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517331880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993204480:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.049306:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002444032 00000100:00100000:0.0:1783520936.049309:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002444160 00000100:00100000:1.0:1783520936.049312:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002444032 00000100:00100000:0.0:1783520936.049314:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002444160 00000100:00100000:2.0:1783520936.049325:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.049336:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002251ab7d/0xd5401cbabc020ea4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x76d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19bc expref: 409 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.049338:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000022f7e7e4/0xd5401cbabc022954 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x76e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3044 expref: 409 pid: 8410 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.049344:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363d3a6300/2, rc 0, transno 4294976169 00000100:00100000:2.0:1783520936.049356:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993202560:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.049407:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36259 00000100:00100000:1.0:1783520936.049420:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357c850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+409:2387:x1870153002444032:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.049424:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36260 00010000:00010000:1.0:1783520936.049434:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.049436:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b6300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+409:2390:x1870153002444160:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.049443:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002251ab7d/0xd5401cbabc020ea4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x76d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19bc expref: 409 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.049447:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993204608:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.049448:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.049457:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000022f7e7e4/0xd5401cbabc022954 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x76e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3044 expref: 409 pid: 8410 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.049467:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002251ab7d/0xd5401cbabc020ea4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x76d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19bc expref: 409 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.049471:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000022f7e7e4/0xd5401cbabc022954 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x76e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3044 expref: 409 pid: 8410 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.049485:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.049487:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002251ab7d/0xd5401cbabc020ea4 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x76d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19bc expref: 409 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.049503:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x76e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.049507:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x76e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.049511:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026211 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.049512:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x76d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.049520:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026211 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x76e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.049521:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x76d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.049532:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02620a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.049533:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993204480 00000100:00100000:2.0:1783520936.049538:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993204480 00010000:00010000:0.0:1783520936.049540:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.049547:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02620a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x76d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.049560:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993204608 00000100:00100000:2.0:1783520936.049563:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993204608 00010000:00010000:1.0:1783520936.049572:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.049577:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b6300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+408:2390:x1870153002444160:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 140us (297us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.049579:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44673 00000100:00100000:2.0:1783520936.049588:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517332a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+601:3625:x1870152993204480:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.049592:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36260 00000100:00100000:1.0:1783520936.049599:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357c850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+408:2387:x1870153002444032:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 181us (407us total) trans 0 rc 0/0 00002000:00080000:2.0:1783520936.049602:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1907 count 1 00000100:00100000:1.0:1783520936.049611:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36259 00010000:00010000:2.0:1783520936.049617:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02622d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x773:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.049626:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44674 00010000:00010000:2.0:1783520936.049629:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000001cbf56/0xd5401cbabc020ec7 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x773:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a10 expref: 406 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.049635:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x773:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.049638:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x773:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.049640:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517331f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+601:3626:x1870152993204608:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.049641:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02622d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x773:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.049661:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1908 count 1 00010000:00010000:2.0:1783520936.049669:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000001cbf56/0xd5401cbabc020ec7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x773:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a10 expref: 406 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.049677:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026234 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x774:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.049678:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000001cbf56/0xd5401cbabc020ec7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x773:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a10 expref: 406 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.049701:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517333b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993204736:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.049706:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002f8c25f9/0xd5401cbabc020ece lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x774:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a1e expref: 406 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.049715:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x774:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.049719:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x774:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.049722:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026234 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x774:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.049739:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967a80 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.049746:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002f8c25f9/0xd5401cbabc020ece lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x774:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a1e expref: 406 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.049753:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02620a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.049758:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002f8c25f9/0xd5401cbabc020ece lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x774:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a1e expref: 406 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.049762:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02620a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.049779:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02620a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.049784:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b5500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993204864:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.049786:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02620a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x76d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.049808:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.049819:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.049838:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.049847:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02620a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x76d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.049849:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993203968:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.049861:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000022f7e7e4/0xd5401cbabc022954 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x76e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3044 expref: 406 pid: 8410 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.049863:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.049866:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026218 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x76f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.049884:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b5180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993203456:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.049900:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+602:3628:x1870152993203072:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3301us (3380us total) trans 4294976170 rc 0/0 00010000:00010000:0.0:1783520936.049901:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026211 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.049914:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44668 00010000:00010000:0.0:1783520936.049919:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026211 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.049928:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026211 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.049942:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026211 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.049945:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b4380/2, rc 0, transno 4294976170 00010000:00010000:0.0:1783520936.049954:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026211 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x76e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.049958:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993203072:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.049984:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.050005:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44675 00010000:00010000:1.0:1783520936.050005:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.050011:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026211 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x76e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.050016:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517333480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+601:3628:x1870152993204352:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.050019:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02620a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x76d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.050027:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.050031:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026211 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x76e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.050035:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1906 count 1 00010000:00010000:3.0:1783520936.050036:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02620a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x76d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.050039:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026211 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x76e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.050051:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02623b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x772:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.050067:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026211 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x76e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.050069:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008c342df2/0xd5401cbabc020ec0 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x772:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a02 expref: 405 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.050075:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+602:3628:x1870152993203328:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3183us (3243us total) trans 4294976171 rc 0/0 00010000:00010000:1.0:1783520936.050091:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026211 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x76e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.050092:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x772:0x0].0x0 since=37s v=(0 0) 00000100:00100000:0.0:1783520936.050095:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44669 00010000:00010000:2.0:1783520936.050100:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x772:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.050109:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02623b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x772:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.050110:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02620a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x76d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.050119:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.050123:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.050127:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.050133:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02620a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x76d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.050133:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b7b80/2, rc 0, transno 4294976171 00000100:00100000:0.0:1783520936.050145:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993203328:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.050146:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008c342df2/0xd5401cbabc020ec0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x772:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a02 expref: 405 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.050159:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.050161:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008c342df2/0xd5401cbabc020ec0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x772:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a02 expref: 405 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.050163:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.050167:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.050170:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.050181:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517332680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993204992:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.050194:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002444288 00000100:00100000:0.0:1783520936.050198:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002444288 00010000:00010000:0.0:1783520936.050212:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000019519042/0xd5401cbabc020eab lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x76f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19d8 expref: 406 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.050217:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895c00 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.050231:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36261 00000100:00100000:0.0:1783520936.050237:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+406:2389:x1870153002444288:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.050243:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.050246:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000019519042/0xd5401cbabc020eab lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x76f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19d8 expref: 406 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.050252:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000019519042/0xd5401cbabc020eab lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x76f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19d8 expref: 406 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.050264:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000019519042/0xd5401cbabc020eab lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x76f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19d8 expref: 406 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.050272:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x76f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.050274:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x76f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.050277:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026218 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.050284:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026218 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x76f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.050301:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.050315:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026218 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.050324:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026218 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.050329:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+405:2389:x1870153002444288:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 89us (165us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.050339:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026218 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x76f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.050341:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36261 00010000:00010000:1.0:1783520936.050386:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026218 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x76f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.050404:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.050483:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517332300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+602:3626:x1870152993203584:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2070us (2225us total) trans 4294976172 rc 0/0 00000100:00100000:1.0:1783520936.050502:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44670 00000004:00080000:2.0:1783520936.050510:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517330000/2, rc 0, transno 4294976172 00000100:00100000:2.0:1783520936.050518:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517330000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993203584:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.050666:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993205120:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.050691:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350ade1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993205248:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.050694:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.050728:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993205376:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.050733:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.050750:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.050757:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026218 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x76f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.050769:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb200 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520936.050769:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.050776:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026218 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x76f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.050789:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026218 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x76f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.050795:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993205120 00000100:00100000:2.0:1783520936.050800:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993205120 00010000:00010000:3.0:1783520936.050803:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026218 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x76f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.050812:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026218 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x76f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.050815:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993205248 00000100:00100000:1.0:1783520936.050821:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993205248 00000100:00100000:2.0:1783520936.050829:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993205376 00010000:00010000:3.0:1783520936.050834:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.050835:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993205376 00010000:00010000:3.0:1783520936.050838:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.050842:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.050857:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44676 00000100:00100000:3.0:1783520936.050869:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44677 00000100:00100000:1.0:1783520936.050870:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517332d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+604:3627:x1870152993205120:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.050878:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44678 00000100:00100000:3.0:1783520936.050881:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+604:3628:x1870152993205248:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.050888:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1909 count 1 00000100:00100000:0.0:1783520936.050892:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+604:3625:x1870152993205376:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520936.050897:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1910 count 1 00010000:00010000:1.0:1783520936.050904:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026242 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x775:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.050909:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1911 count 1 00010000:00010000:3.0:1783520936.050912:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026249 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x776:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.050919:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005b8efbfe/0xd5401cbabc020ed5 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x775:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a2c expref: 404 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.050926:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026250 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x777:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.050931:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000003136cec/0xd5401cbabc022bae lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x776:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f329e expref: 404 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.050934:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x775:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.050943:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x775:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.050953:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x776:0x0].0x0 since=36s v=(0 0) 00010000:00010000:1.0:1783520936.050954:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026242 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x775:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.050954:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008f726561/0xd5401cbabc020ee3 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x777:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a48 expref: 404 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.050960:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x776:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.050970:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026249 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x776:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.050981:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x777:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.050990:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x777:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.050999:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026250 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x777:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.051014:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005b8efbfe/0xd5401cbabc020ed5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x775:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a2c expref: 404 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.051022:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000003136cec/0xd5401cbabc022bae lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x776:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f329e expref: 404 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.051027:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008f726561/0xd5401cbabc020ee3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x777:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a48 expref: 404 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.051030:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005b8efbfe/0xd5401cbabc020ed5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x775:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a2c expref: 404 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.051046:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000003136cec/0xd5401cbabc022bae lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x776:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f329e expref: 404 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.051053:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008f726561/0xd5401cbabc020ee3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x777:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a48 expref: 404 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.051059:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a4e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993205504:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.051071:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032f3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993205632:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.051077:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b4000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993205760:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.051114:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3eb40 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.051116:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c3696c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.051301:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c016cc0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.052910:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b5880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993204224:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.052923:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993204096:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.052935:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.052938:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026226 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x770:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.052947:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02621f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x771:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.052958:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517333b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993204736:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.052971:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.052980:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02622d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x773:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.052994:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002444416 00000100:00100000:2.0:1783520936.052999:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002444416 00000100:00100000:0.0:1783520936.053002:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.053018:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002444544 00010000:00010000:2.0:1783520936.053023:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e8a94b3e/0xd5401cbabc020eb2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x770:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19e6 expref: 405 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.053025:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002444544 00000100:00100000:1.0:1783520936.053035:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002444672 00000100:00100000:1.0:1783520936.053040:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002444672 00010000:00010000:3.0:1783520936.053050:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000043dbd9a7/0xd5401cbabc020eb9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x771:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19f4 expref: 407 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.053058:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36262 00010000:00010000:1.0:1783520936.053064:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000001cbf56/0xd5401cbabc020ec7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x773:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a10 expref: 407 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.053071:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b4700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+407:2387:x1870153002444416:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.053081:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.053087:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.053092:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e8a94b3e/0xd5401cbabc020eb2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x770:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19e6 expref: 407 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.053100:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e8a94b3e/0xd5401cbabc020eb2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x770:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19e6 expref: 407 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.053101:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002444800 00000100:00100000:3.0:1783520936.053107:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002444800 00000100:00100000:1.0:1783520936.053110:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36263 00000100:00100000:1.0:1783520936.053121:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b6680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+408:2387:x1870153002444544:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.053123:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e8a94b3e/0xd5401cbabc020eb2 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x770:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19e6 expref: 408 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.053128:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002f8c25f9/0xd5401cbabc020ece lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x774:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a1e expref: 408 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.053133:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.053144:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.053155:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000043dbd9a7/0xd5401cbabc020eb9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x771:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19f4 expref: 408 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.053162:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002444928 00010000:00010000:2.0:1783520936.053163:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x770:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.053168:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002444928 00010000:00010000:2.0:1783520936.053173:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x770:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.053180:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.053184:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026226 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x770:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.053185:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000043dbd9a7/0xd5401cbabc020eb9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x771:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19f4 expref: 408 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.053188:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008c342df2/0xd5401cbabc020ec0 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x772:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a02 expref: 408 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.053215:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002445056 00010000:00010000:2.0:1783520936.053217:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026226 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x770:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.053220:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002445056 00000100:00100000:0.0:1783520936.053220:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.053221:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000043dbd9a7/0xd5401cbabc020eb9 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x771:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f19f4 expref: 408 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.053241:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x771:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.053242:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008f726561/0xd5401cbabc020ee3 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x777:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a48 expref: 408 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.053248:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x771:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.053256:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.053258:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02621f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x771:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.053264:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.053266:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002445184 00000100:00100000:3.0:1783520936.053271:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002445184 00010000:00010000:1.0:1783520936.053274:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02621f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x771:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.053282:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002445312 00000100:00100000:0.0:1783520936.053287:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002445312 00010000:00010000:3.0:1783520936.053295:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000003136cec/0xd5401cbabc022bae lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x776:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f329e expref: 410 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.053297:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b4700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+410:2387:x1870153002444416:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 228us (362us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.053297:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.053303:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005b8efbfe/0xd5401cbabc020ed5 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x775:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a2c expref: 410 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.053316:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36262 00000100:00100000:1.0:1783520936.053325:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b6680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+410:2387:x1870153002444544:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 203us (353us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.053334:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36263 00000100:00100000:3.0:1783520936.053341:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36264 00000100:00100000:2.0:1783520936.053349:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36265 00000100:00100000:3.0:1783520936.053354:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b4a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+408:2389:x1870153002444672:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.053383:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36266 00000100:00100000:2.0:1783520936.053386:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b5880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+408:2388:x1870153002444800:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.053391:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.053396:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357f850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+408:2390:x1870153002444928:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.053397:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36267 00010000:00010000:2.0:1783520936.053398:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.053402:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000001cbf56/0xd5401cbabc020ec7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x773:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a10 expref: 408 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.053409:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002f8c25f9/0xd5401cbabc020ece lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x774:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a1e expref: 408 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.053410:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.053415:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+408:2387:x1870153002445056:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.053418:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008c342df2/0xd5401cbabc020ec0 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x772:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a02 expref: 408 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.053423:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000001cbf56/0xd5401cbabc020ec7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x773:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a10 expref: 408 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.053431:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.053441:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008f726561/0xd5401cbabc020ee3 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x777:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a48 expref: 408 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.053442:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008c342df2/0xd5401cbabc020ec0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x772:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a02 expref: 408 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.053454:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000001cbf56/0xd5401cbabc020ec7 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x773:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a10 expref: 408 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.053465:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002f8c25f9/0xd5401cbabc020ece lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x774:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a1e expref: 408 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.053471:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x772:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.053479:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x772:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.053483:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x773:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.053489:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02623b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x772:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.053509:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x773:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.053511:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x774:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.053515:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008f726561/0xd5401cbabc020ee3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x777:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a48 expref: 407 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.053519:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02623b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x772:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.053520:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02622d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x773:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.053520:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x774:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.053532:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026234 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x774:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.053545:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x777:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.053547:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02622d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x773:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.053552:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026234 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x774:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.053553:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x777:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.053554:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.053564:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026250 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x777:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.053576:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357f850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+407:2390:x1870153002444928:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 184us (448us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.053577:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026250 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x777:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.053579:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.053583:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.053588:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36266 00010000:00010000:1.0:1783520936.053605:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.053607:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36268 00000100:00100000:3.0:1783520936.053614:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b4a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+406:2389:x1870153002444672:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 263us (612us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.053620:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+406:2389:x1870153002445184:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.053622:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b5880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+406:2388:x1870153002444800:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 260us (540us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.053634:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+406:2387:x1870153002445056:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 222us (454us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.053635:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36264 00010000:00010000:0.0:1783520936.053640:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.053648:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000003136cec/0xd5401cbabc022bae lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x776:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f329e expref: 406 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.053649:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36265 00000100:00100000:1.0:1783520936.053659:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36267 00010000:00010000:0.0:1783520936.053660:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000003136cec/0xd5401cbabc022bae lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x776:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f329e expref: 405 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.053668:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36269 00010000:00010000:0.0:1783520936.053677:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x776:0x0].0x0 since=0s v=(1 0) 00000100:00100000:3.0:1783520936.053683:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+403:2388:x1870153002445312:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.053683:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x776:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.053689:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02621f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x771:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.053693:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026249 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x776:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.053699:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.053702:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026226 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x770:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.053709:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005b8efbfe/0xd5401cbabc020ed5 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x775:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a2c expref: 403 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.053710:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02621f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x771:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.053717:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026249 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x776:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.053730:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026226 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x770:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.053742:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02621f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x771:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.053743:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005b8efbfe/0xd5401cbabc020ed5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x775:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a2c expref: 403 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.053747:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.053754:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026226 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x770:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.053765:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514ff2d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+403:2389:x1870153002445184:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 149us (549us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.053770:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02621f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x771:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.053776:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x775:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.053779:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026226 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x770:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.053782:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36268 00010000:00010000:3.0:1783520936.053785:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x775:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.053797:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026242 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x775:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.053807:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026242 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x775:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.053814:0:8410:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002f8c25f9/0xd5401cbabc020ece lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x774:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a1e expref: 402 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.053818:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520936.053826:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.053827:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b5500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993204864:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.053829:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520936.053836:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520936.053844:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.053845:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02621f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x771:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.053845:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026234 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x774:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.053852:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026226 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x770:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.053853:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+401:2388:x1870153002445312:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 171us (589us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.053872:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbaa00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:3.0:1783520936.053874:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36269 00010000:00010000:1.0:1783520936.053880:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02621f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x771:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.053881:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.053887:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026234 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x774:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.053889:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026226 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x770:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.053896:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02621f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x771:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.053911:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026234 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x774:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.053914:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026226 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x770:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.053916:0:35275:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000003136cec/0xd5401cbabc022bae lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x776:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f329e expref: 400 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.053922:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02621f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x771:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.053938:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026234 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x774:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.053946:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026226 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x770:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.053947:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02621f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x771:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.053949:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032f3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993205632:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.053958:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026234 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x774:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.053968:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026249 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x776:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.053972:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026226 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x770:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.053985:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.053990:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.053996:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026234 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x774:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.053997:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to regular blp list, 0 blwis in pool 00010000:00010000:3.0:1783520936.054002:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.054004:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.054007:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.054010:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.054010:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.054012:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026249 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x776:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.054018:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026234 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x774:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.054028:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026249 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x776:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.054033:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026234 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x774:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.054047:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026249 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x776:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.054052:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026234 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x774:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.054061:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026249 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x776:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.054064:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026234 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x774:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.054097:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.054100:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to regular blp list, 0 blwis in pool 00010000:00010000:1.0:1783520936.054103:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026249 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x776:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.054110:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.054111:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.054117:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026249 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x776:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.054125:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026249 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x776:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.054136:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026249 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x776:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.054137:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517330380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+606:3625:x1870152993203712:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5311us (5513us total) trans 4294976173 rc 0/0 00000100:00100000:0.0:1783520936.054147:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517331f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+606:3626:x1870152993204608:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4509us (4663us total) trans 4294976174 rc 0/0 00000100:00100000:2.0:1783520936.054152:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44672 00010000:00010000:1.0:1783520936.054160:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026249 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x776:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.054165:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44674 00000004:00080000:2.0:1783520936.054188:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350ade0700/2, rc 0, transno 4294976174 00010000:00010000:1.0:1783520936.054191:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.054194:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.054196:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.054203:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993204608:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.054222:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008f726561/0xd5401cbabc020ee3 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x777:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a48 expref: 399 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.054225:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517330a80/2, rc 0, transno 4294976173 00000100:00100000:3.0:1783520936.054230:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+605:3628:x1870152993205248:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3350us (3495us total) trans 4294976175 rc 0/0 00000100:00100000:1.0:1783520936.054237:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517330a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993203712:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.054240:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b4000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993205760:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.054245:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44677 00010000:00010000:0.0:1783520936.054263:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026250 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x777:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.054289:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350ade1f80/2, rc 0, transno 4294976175 00000100:00100000:0.0:1783520936.054296:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993205248:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.054318:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026250 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x777:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.054324:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993205888:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.054330:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026250 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x777:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.054342:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026250 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x777:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.054351:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026250 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x777:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.054384:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.054390:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517333100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+605:3628:x1870152993203840:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5592us (5840us total) trans 4294976176 rc 0/0 00000100:00100000:3.0:1783520936.054396:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993205888 00000100:00100000:1.0:1783520936.054401:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44671 00000100:00100000:3.0:1783520936.054402:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993205888 00000100:00100000:0.0:1783520936.054427:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993206144:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.054435:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993206016:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.054442:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44679 00000100:00100000:1.0:1783520936.054452:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517333b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+605:3626:x1870152993205888:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.054459:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.054460:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02622d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x773:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.054469:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02622d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x773:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.054470:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1912 count 1 00010000:00010000:3.0:1783520936.054481:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02622d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x773:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.054481:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993206144 00010000:00010000:1.0:1783520936.054486:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026257 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x778:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.054488:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993206144 00010000:00010000:3.0:1783520936.054510:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02622d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x773:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.054513:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517330700/2, rc 0, transno 4294976176 00010000:00010000:1.0:1783520936.054517:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008c356d1f/0xd5401cbabc020eea lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x778:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a56 expref: 398 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.054529:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.054532:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517330700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993203840:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.054538:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x778:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.054543:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x778:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.054548:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026257 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x778:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.054561:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.054582:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993206272:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.054595:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.054601:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026250 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x777:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.054610:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea600 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520936.054612:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993206016 00010000:00010000:3.0:1783520936.054616:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026250 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x777:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.054617:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.054618:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993206016 00010000:00010000:3.0:1783520936.054629:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026250 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x777:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.054644:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026250 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x777:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.054646:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993206272 00000100:00100000:2.0:1783520936.054652:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993206272 00000100:00100000:0.0:1783520936.054652:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+608:3625:x1870152993205376:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3762us (3883us total) trans 4294976177 rc 0/0 00010000:00010000:3.0:1783520936.054657:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026250 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x777:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.054657:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435032f0e00/2, rc 0, transno 4294976177 00000100:00100000:0.0:1783520936.054666:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44678 00000100:00100000:1.0:1783520936.054673:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993205376:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.054686:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.054689:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.054693:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.054701:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44680 00000100:00100000:0.0:1783520936.054709:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44681 00000100:00100000:2.0:1783520936.054713:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351dc40700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+608:3628:x1870152993206144:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.054720:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44682 00000100:00100000:0.0:1783520936.054722:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+608:3627:x1870152993206016:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.054730:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1914 count 1 00000100:00100000:1.0:1783520936.054733:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1783520936.054735:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+608:3625:x1870152993206272:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.054739:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1913 count 1 00000100:00100000:1.0:1783520936.054744:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993206400:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.054747:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02625e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x77a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.054753:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1915 count 1 00010000:00010000:0.0:1783520936.054755:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026265 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x779:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.054765:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a9e8dd06/0xd5401cbabc020ef8 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x77a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a72 expref: 398 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.054769:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02626c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x77b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.054773:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f1fe65ed/0xd5401cbabc020ef1 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x779:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a64 expref: 398 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.054780:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.054781:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x77a:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.054791:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x77a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.054798:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x779:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.054800:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d0cbfffd/0xd5401cbabc020eff lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x77b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a80 expref: 398 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.054801:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02625e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x77a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.054805:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x779:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.054809:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993206400 00000100:00100000:1.0:1783520936.054815:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993206400 00010000:00010000:0.0:1783520936.054817:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026265 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x779:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.054824:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x77b:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.054831:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x77b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.054838:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02626c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x77b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.054839:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44683 00010000:00010000:2.0:1783520936.054844:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a9e8dd06/0xd5401cbabc020ef8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x77a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a72 expref: 398 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.054852:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f1fe65ed/0xd5401cbabc020ef1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x779:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a64 expref: 398 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.054854:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515776680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+609:3625:x1870152993206400:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.054877:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a9e8dd06/0xd5401cbabc020ef8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x77a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a72 expref: 398 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.054880:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f1fe65ed/0xd5401cbabc020ef1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x779:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a64 expref: 398 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.054892:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1916 count 1 00010000:00010000:3.0:1783520936.054896:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d0cbfffd/0xd5401cbabc020eff lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x77b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a80 expref: 398 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.054903:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351dc41500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993206656:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.054909:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026273 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x77c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.054910:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b5500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993206528:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.054915:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d0cbfffd/0xd5401cbabc020eff lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x77b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a80 expref: 398 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.054935:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000deaf78cd/0xd5401cbabc020f06 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x77c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a8e expref: 398 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.054940:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e3b3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993206784:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.054944:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.054946:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f180 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.054952:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x77c:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.054956:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x77c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.054960:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026273 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x77c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.054980:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967a80 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.054986:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000deaf78cd/0xd5401cbabc020f06 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x77c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a8e expref: 398 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.054988:0:20415:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008c342df2/0xd5401cbabc020ec0 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x772:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a02 expref: 398 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.055003:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.055009:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.055010:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000deaf78cd/0xd5401cbabc020f06 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x77c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a8e expref: 398 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.055015:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02622d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x773:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.055015:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517332680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993204992:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.055034:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.055039:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02622d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x773:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.055041:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02623b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x772:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.055047:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993206912:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.055050:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02622d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x773:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.055065:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02623b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x772:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.055073:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02622d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x773:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.055075:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02623b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x772:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.055076:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517332a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+610:3625:x1870152993204480:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5488us (5750us total) trans 4294976178 rc 0/0 00000100:00100000:1.0:1783520936.055084:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514825600 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.055092:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44673 00010000:00010000:2.0:1783520936.055100:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02622d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x773:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.055101:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02623b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x772:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.055106:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008c356d1f/0xd5401cbabc020eea lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x778:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a56 expref: 397 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.055122:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517331880/2, rc 0, transno 4294976178 00010000:00010000:0.0:1783520936.055123:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02623b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x772:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.055133:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008c356d1f/0xd5401cbabc020eea lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x778:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a56 expref: 397 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.055137:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.055141:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to regular blp list, 0 blwis in pool 00000100:00100000:3.0:1783520936.055142:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517331880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993204480:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.055145:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02623b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x772:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.055159:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520936.055162:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993207040:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.055163:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02623b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x772:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.055164:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.055175:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02623b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x772:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.055187:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02623b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x772:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.055196:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02623b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x772:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.055216:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.055218:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.055222:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363840ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993207168:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.055222:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.055229:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517333480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+610:3628:x1870152993204352:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5212us (6139us total) trans 4294976179 rc 0/0 00000100:00100000:0.0:1783520936.055238:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44675 00000100:00100000:1.0:1783520936.055242:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435192190c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.055260:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.055281:0:178496:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005b8efbfe/0xd5401cbabc020ed5 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x775:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a2c expref: 397 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.055285:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517333800/2, rc 0, transno 4294976179 00000100:00100000:1.0:1783520936.055302:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a4e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993205504:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.055306:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517333800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993204352:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.055321:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026242 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x775:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.055327:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993207168 00000100:00100000:0.0:1783520936.055331:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993207168 00000100:00100000:1.0:1783520936.055404:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1783520936.055415:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363840c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993207296:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.055425:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44684 00000100:00100000:0.0:1783520936.055435:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e3b1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+610:3627:x1870152993207168:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.055445:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00080000:0.0:1783520936.055449:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1917 count 1 00010000:00010000:0.0:1783520936.055463:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02627a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x77d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.055475:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000051da37f/0xd5401cbabc020f0d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x77d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a9c expref: 396 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.055478:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026242 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x775:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.055483:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993207296 00000100:00100000:2.0:1783520936.055489:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993207296 00010000:00010000:0.0:1783520936.055516:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x77d:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.055518:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026242 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x775:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.055522:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x77d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.055532:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02627a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x77d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.055533:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44685 00010000:00010000:1.0:1783520936.055533:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026242 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x775:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.055548:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363840df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+611:3627:x1870152993207296:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.055558:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026242 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x775:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.055567:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1918 count 1 00010000:00010000:0.0:1783520936.055568:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000051da37f/0xd5401cbabc020f0d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x77d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a9c expref: 396 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.055575:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.055580:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026281 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x77e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.055584:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000051da37f/0xd5401cbabc020f0d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x77d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a9c expref: 396 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.055587:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.055596:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000088b2caeb/0xd5401cbabc022a9d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x77e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f318d expref: 396 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.055596:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026242 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x775:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.055605:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517332680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993207424:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.055622:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x77e:0x0].0x0 since=36s v=(0 0) 00010000:00010000:1.0:1783520936.055622:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.055628:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026242 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x775:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.055629:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x77e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.055637:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026281 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x77e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.055643:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026242 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x775:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.055644:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.055661:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026242 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x775:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.055666:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000088b2caeb/0xd5401cbabc022a9d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x77e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f318d expref: 396 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.055676:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026242 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x775:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.055683:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000088b2caeb/0xd5401cbabc022a9d lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x77e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f318d expref: 396 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.055699:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b6300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993207552:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.055700:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.055705:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.055708:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.055726:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872b40 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.055933:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517332d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+612:3627:x1870152993205120:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5069us (5239us total) trans 4294976180 rc 0/0 00000100:00100000:1.0:1783520936.055938:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44676 00000004:00080000:3.0:1783520936.055956:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350ade1880/2, rc 0, transno 4294976180 00000100:00100000:3.0:1783520936.055970:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350ade1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993205120:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.056021:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363840ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993207680:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.056054:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.056138:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993207680 00000100:00100000:0.0:1783520936.056143:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993207680 00000100:00100000:0.0:1783520936.056170:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44686 00000100:00100000:0.0:1783520936.056177:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e3b3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+612:3625:x1870152993207680:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.056201:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1919 count 1 00010000:00010000:0.0:1783520936.056213:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026288 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x77f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.056225:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001d8ca43d/0xd5401cbabc020f22 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x77f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ab8 expref: 396 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.056237:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x77f:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.056241:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x77f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.056245:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026288 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x77f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.056266:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001d8ca43d/0xd5401cbabc020f22 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x77f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ab8 expref: 396 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.056277:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001d8ca43d/0xd5401cbabc020f22 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x77f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ab8 expref: 396 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.056288:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503324700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993207808:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.056324:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3eb40 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.057852:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b5500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993206528:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.057856:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.057874:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351dc41500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993206656:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.057879:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02625e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x77a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.057889:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.057896:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026265 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x779:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.057911:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002445568 00000100:00100000:2.0:1783520936.057917:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002445568 00010000:00010000:2.0:1783520936.057936:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f1fe65ed/0xd5401cbabc020ef1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x779:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a64 expref: 397 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.057957:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002445440 00000100:00100000:2.0:1783520936.057960:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002445440 00010000:00010000:2.0:1783520936.057970:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a9e8dd06/0xd5401cbabc020ef8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x77a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a72 expref: 398 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.057977:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36270 00000100:00100000:0.0:1783520936.057985:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b4700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+398:2387:x1870153002445568:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.057993:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36271 00010000:00010000:0.0:1783520936.057996:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.057999:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b6680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+398:2387:x1870153002445440:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.058002:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f1fe65ed/0xd5401cbabc020ef1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x779:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a64 expref: 398 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.058005:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.058010:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a9e8dd06/0xd5401cbabc020ef8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x77a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a72 expref: 398 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.058022:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a9e8dd06/0xd5401cbabc020ef8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x77a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a72 expref: 398 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.058028:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f1fe65ed/0xd5401cbabc020ef1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x779:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a64 expref: 398 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.058041:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a9e8dd06/0xd5401cbabc020ef8 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x77a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a72 expref: 398 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.058041:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f1fe65ed/0xd5401cbabc020ef1 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x779:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a64 expref: 398 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.058055:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x77a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.058055:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x779:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.058059:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x77a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.058059:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x779:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.058065:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02625e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.058065:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026265 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x779:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.058075:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02625e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x77a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.058075:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026265 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x779:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.058095:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.058102:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.058107:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026265 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x779:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.058118:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026265 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x779:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.058130:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b4700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+396:2387:x1870153002445568:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 143us (273us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.058131:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026265 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x779:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.058131:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b6680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+396:2387:x1870153002445440:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 130us (239us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.058146:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36271 00010000:00010000:3.0:1783520936.058149:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026265 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x779:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.058154:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36270 00010000:00010000:3.0:1783520936.058173:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.058181:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02625e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.058185:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.058192:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02625e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.058194:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026265 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x779:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.058204:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02625e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.058209:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.058212:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02625e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x77a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.058215:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026265 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x779:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.058227:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026265 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x779:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.058232:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.058241:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.058242:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026265 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x779:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.058249:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02625e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x77a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.058255:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026265 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x779:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.058266:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.058271:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02625e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x77a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.058273:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993206912:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.058280:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.058285:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.058287:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02625e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x77a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.058289:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.058291:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026273 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x77c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.058303:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02625e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x77a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.058310:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02625e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x77a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.058333:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.058336:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.058338:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.058407:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351dc40700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+614:3628:x1870152993206144:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3697us (3951us total) trans 4294976181 rc 0/0 00000100:00100000:1.0:1783520936.058414:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44680 00000004:00080000:0.0:1783520936.058417:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435032f0000/2, rc 0, transno 4294976181 00000100:00100000:0.0:1783520936.058432:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993206144:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.058435:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+613:3627:x1870152993206016:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3715us (3908us total) trans 4294976182 rc 0/0 00000100:00100000:3.0:1783520936.058444:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44681 00000004:00080000:2.0:1783520936.058457:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435032f1500/2, rc 0, transno 4294976182 00000100:00100000:2.0:1783520936.058464:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993206016:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.058483:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e3b3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993206784:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.058512:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:3.0:1783520936.058515:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02626c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x77b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.058522:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1783520936.058524:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503327800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993207936:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.058530:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503327100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993208064:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.058558:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.058578:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.058592:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993208064 00000100:00100000:3.0:1783520936.058597:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993208064 00000100:00100000:3.0:1783520936.058627:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993207936 00000100:00100000:3.0:1783520936.058630:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993207936 00000100:00100000:3.0:1783520936.058659:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44687 00000100:00100000:0.0:1783520936.058927:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44688 00000100:00100000:3.0:1783520936.058932:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503325880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+614:3628:x1870152993208064:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.058942:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+614:3625:x1870152993207936:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520936.058954:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1921 count 1 00002000:00080000:0.0:1783520936.058959:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1920 count 1 00000100:00100000:1.0:1783520936.058972:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.058978:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02628f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x781:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.058978:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026296 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x780:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.059004:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c0e509b3/0xd5401cbabc020f1b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x780:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ac6 expref: 394 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.059005:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000091926ded/0xd5401cbabc020f29 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x781:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ad4 expref: 394 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.059030:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.059041:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x780:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.059043:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x781:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.059046:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x780:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.059049:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x781:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.059053:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026296 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x780:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.059055:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02628f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x781:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.059093:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c0e509b3/0xd5401cbabc020f1b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x780:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ac6 expref: 394 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.059096:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000091926ded/0xd5401cbabc020f29 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x781:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ad4 expref: 394 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.059100:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.059123:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c0e509b3/0xd5401cbabc020f1b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x780:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ac6 expref: 394 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.059124:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000091926ded/0xd5401cbabc020f29 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x781:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ad4 expref: 394 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.059131:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002445696 00000100:00100000:2.0:1783520936.059136:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002445696 00000100:00100000:1.0:1783520936.059149:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.059151:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503325c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993208192:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.059157:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e3b3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993208320:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.059158:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d0cbfffd/0xd5401cbabc020eff lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x77b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a80 expref: 395 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.059184:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002445824 00000100:00100000:2.0:1783520936.059188:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002445824 00000100:00100000:1.0:1783520936.059198:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.059203:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000deaf78cd/0xd5401cbabc020f06 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x77c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a8e expref: 396 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.059207:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.059210:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c016cc0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.059224:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002445952 00000100:00100000:0.0:1783520936.059225:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002446080 00000100:00100000:2.0:1783520936.059229:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002445952 00000100:00100000:0.0:1783520936.059229:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002446080 00010000:00010000:2.0:1783520936.059243:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008c356d1f/0xd5401cbabc020eea lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x778:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a56 expref: 398 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.059244:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000088b2caeb/0xd5401cbabc022a9d lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x77e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f318d expref: 398 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.059257:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002446208 00000100:00100000:0.0:1783520936.059259:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36272 00000100:00100000:2.0:1783520936.059260:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002446208 00000100:00100000:0.0:1783520936.059265:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363840e680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+399:2387:x1870153002445696:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.059270:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000051da37f/0xd5401cbabc020f0d lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x77d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a9c expref: 399 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.059273:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.059278:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d0cbfffd/0xd5401cbabc020eff lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x77b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a80 expref: 399 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.059285:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36273 00000100:00100000:1.0:1783520936.059285:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.059295:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357dc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+399:2387:x1870153002445824:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.059298:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d0cbfffd/0xd5401cbabc020eff lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x77b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a80 expref: 399 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.059302:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.059306:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000deaf78cd/0xd5401cbabc020f06 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x77c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a8e expref: 399 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.059312:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d0cbfffd/0xd5401cbabc020eff lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x77b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a80 expref: 399 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.059323:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000deaf78cd/0xd5401cbabc020f06 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x77c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a8e expref: 398 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.059324:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x77b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.059328:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x77b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.059332:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02626c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.059335:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000deaf78cd/0xd5401cbabc020f06 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x77c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a8e expref: 398 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.059340:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02626c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x77b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.059345:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x77c:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.059348:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002446336 00010000:00010000:2.0:1783520936.059349:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x77c:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.059355:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002446336 00010000:00010000:2.0:1783520936.059356:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026273 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.059377:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.059384:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026273 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x77c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.059401:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.059403:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363840e680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+398:2387:x1870153002445696:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 137us (432us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.059411:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001d8ca43d/0xd5401cbabc020f22 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x77f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ab8 expref: 398 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.059413:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36272 00000100:00100000:2.0:1783520936.059419:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357dc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+398:2387:x1870153002445824:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 123us (393us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.059431:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36273 00000100:00100000:0.0:1783520936.059435:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36274 00000100:00100000:0.0:1783520936.059442:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363840d880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+396:2387:x1870153002445952:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.059450:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.059454:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36275 00010000:00010000:0.0:1783520936.059457:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008c356d1f/0xd5401cbabc020eea lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x778:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a56 expref: 396 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.059469:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a4e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+396:2387:x1870153002446080:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.059476:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36276 00010000:00010000:0.0:1783520936.059479:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008c356d1f/0xd5401cbabc020eea lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x778:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a56 expref: 396 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.059484:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.059489:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e472300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+396:2390:x1870153002446208:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.059504:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000088b2caeb/0xd5401cbabc022a9d lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x77e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f318d expref: 396 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.059507:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x778:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.059514:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x778:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.059515:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.059523:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000051da37f/0xd5401cbabc020f0d lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x77d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a9c expref: 396 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.059523:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026257 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x778:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.059530:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000088b2caeb/0xd5401cbabc022a9d lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x77e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f318d expref: 396 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.059536:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026257 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x778:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.059540:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02626c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.059541:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000051da37f/0xd5401cbabc020f0d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x77d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a9c expref: 396 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.059551:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x77e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.059560:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x77e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.059561:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.059566:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02626c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.059569:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026281 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.059572:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x77d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.059580:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x77d:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.059581:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363840d880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+396:2387:x1870153002445952:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 139us (485us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.059587:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02627a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.059588:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02626c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.059597:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36274 00010000:00010000:1.0:1783520936.059605:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026281 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x77e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.059611:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02627a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x77d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.059613:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02626c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x77b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.059615:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36277 00000100:00100000:0.0:1783520936.059627:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e471f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+395:2390:x1870153002446336:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.059639:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.059640:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.059642:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.059647:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001d8ca43d/0xd5401cbabc020f22 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x77f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ab8 expref: 395 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.059657:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001d8ca43d/0xd5401cbabc020f22 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x77f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ab8 expref: 395 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.059660:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.059667:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.059674:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02626c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x77b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.059676:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a4e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+395:2387:x1870153002446080:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 210us (527us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.059680:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x77f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.059686:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x77f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.059691:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.059692:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026288 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.059693:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36275 00010000:00010000:2.0:1783520936.059697:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02626c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x77b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.059705:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026288 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x77f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.059706:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02626c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x77b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.059718:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02626c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x77b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.059720:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.059725:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02626c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x77b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.059736:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008c356d1f/0xd5401cbabc020eea lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x778:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a56 expref: 394 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.059743:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.059743:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e471f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+394:2390:x1870153002446336:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 116us (462us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.059747:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.059751:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.059751:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993207040:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.059752:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36277 00010000:00010000:1.0:1783520936.059774:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026257 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x778:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.059775:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e472300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+392:2390:x1870153002446208:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 288us (577us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.059782:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+615:3625:x1870152993206272:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5050us (5164us total) trans 4294976183 rc 0/0 00010000:00010000:0.0:1783520936.059795:0:8410:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000051da37f/0xd5401cbabc020f0d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x77d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1a9c expref: 392 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.059801:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44682 00000100:00100000:2.0:1783520936.059802:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36276 00010000:00010000:1.0:1783520936.059803:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026257 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x778:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.059815:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517332680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993207424:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.059816:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026257 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x778:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.059827:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026273 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.059829:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026257 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x778:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.059833:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02627a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.059844:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026273 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.059855:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02627a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.059856:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026257 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x778:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.059859:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026273 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.059871:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02627a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.059881:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026273 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x77c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.059888:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.059890:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02627a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.059896:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:0.0:1783520936.059899:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02627a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x77d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.059901:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1783520936.059908:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026257 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x778:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.059916:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520936.059926:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.059928:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1783520936.059930:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026257 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x778:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.059934:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026273 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x77c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.059942:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026257 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x778:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.059947:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.059953:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026273 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x77c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.059960:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026257 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x778:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.059963:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026273 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x77c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.059973:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026257 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x778:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.059983:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515776680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+616:3625:x1870152993206400:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5131us (5205us total) trans 4294976184 rc 0/0 00010000:00010000:0.0:1783520936.059983:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026273 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x77c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.059991:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026273 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x77c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.059992:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44683 00010000:00010000:1.0:1783520936.059994:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.059997:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.060001:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02627a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x77d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.060009:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.060013:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02627a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x77d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.060015:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.060020:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.060023:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02627a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x77d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.060025:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520936.060029:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363d3a4700/2, rc 0, transno 4294976184 00010000:00010000:2.0:1783520936.060033:0:35282:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000088b2caeb/0xd5401cbabc022a9d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x77e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f318d expref: 390 pid: 8409 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.060037:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02627a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x77d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.060044:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993206400:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.060057:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02627a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x77d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.060061:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b6300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993207552:192.168.204.33@tcp:104: 00000004:00080000:3.0:1783520936.060066:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435032f2a00/2, rc 0, transno 4294976183 00000100:00100000:3.0:1783520936.060075:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993206272:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.060082:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026281 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.060088:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.060092:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026281 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.060093:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.060097:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.060106:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026281 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.060114:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e3b1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+615:3627:x1870152993207168:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4681us (4854us total) trans 4294976185 rc 0/0 00010000:00010000:2.0:1783520936.060115:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026281 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.060123:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026281 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x77e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.060123:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44684 00000004:00080000:2.0:1783520936.060152:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363840ce00/2, rc 0, transno 4294976185 00010000:00010000:0.0:1783520936.060155:0:12734:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001d8ca43d/0xd5401cbabc020f22 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x77f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ab8 expref: 389 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.060160:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e3b1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993208448:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.060164:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363840ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993207168:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.060170:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503324700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993207808:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.060186:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026288 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.060193:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517333b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+615:3626:x1870152993205888:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5740us (5807us total) trans 4294976186 rc 0/0 00010000:00010000:0.0:1783520936.060201:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026288 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.060206:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44679 00000100:00100000:3.0:1783520936.060216:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.060217:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026288 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.060228:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026288 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x77f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.060236:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026288 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x77f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.060237:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993208448 00000100:00100000:1.0:1783520936.060242:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993208448 00010000:00010000:0.0:1783520936.060259:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.060269:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44689 00010000:00010000:2.0:1783520936.060273:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520936.060277:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e3b2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+615:3628:x1870152993208448:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.060280:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026288 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x77f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.060284:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e3b3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993208704:0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.060291:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1922 count 1 00010000:00010000:2.0:1783520936.060296:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.060301:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026288 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x77f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.060307:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc02629d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x782:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.060310:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026288 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x77f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.060326:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.060337:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e2b92963/0xd5401cbabc020f37 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x782:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ae2 expref: 388 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.060338:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026288 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x77f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.060338:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e3b3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+616:3625:x1870152993207680:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4159us (4281us total) trans 4294976187 rc 0/0 00000100:00100000:0.0:1783520936.060354:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44686 00000100:00100000:0.0:1783520936.060401:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993208704 00010000:00010000:1.0:1783520936.060402:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x782:0x0].0x0 since=37s v=(0 0) 00000100:00100000:0.0:1783520936.060406:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993208704 00000004:00080000:3.0:1783520936.060408:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435032f3480/2, rc 0, transno 4294976186 00010000:00010000:1.0:1783520936.060408:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x782:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.060418:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc02629d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x782:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.060422:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026288 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x77f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.060425:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993205888:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.060433:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44690 00000100:00100000:3.0:1783520936.060441:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:1.0:1783520936.060442:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e2b92963/0xd5401cbabc020f37 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x782:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ae2 expref: 388 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.060443:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e3b1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+616:3626:x1870152993208704:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.060444:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.060447:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.060450:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.060452:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e3b2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993208576:0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.060459:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1924 count 1 00010000:00010000:1.0:1783520936.060462:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e2b92963/0xd5401cbabc020f37 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x782:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ae2 expref: 388 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.060471:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.060476:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e471c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993208832:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.060476:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0262a4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x784:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.060477:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.060484:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026281 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x77e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.060512:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.060515:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000baf78fb0/0xd5401cbabc020f3e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x784:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1afe expref: 388 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.060518:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026281 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x77e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.060532:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x784:0x0].0x0 since=37s v=(0 0) 00000100:00100000:1.0:1783520936.060533:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c3696c0 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.060537:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026281 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x77e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.060540:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x784:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.060550:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0262a4 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x784:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.060558:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363840df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+617:3627:x1870152993207296:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5015us (5114us total) trans 4294976188 rc 0/0 00000100:00100000:2.0:1783520936.060565:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44685 00010000:00010000:0.0:1783520936.060571:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000baf78fb0/0xd5401cbabc020f3e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x784:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1afe expref: 388 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.060582:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000baf78fb0/0xd5401cbabc020f3e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x784:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1afe expref: 388 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.060593:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503324700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993208960:192.168.204.33@tcp:104: 00000004:00080000:2.0:1783520936.060596:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363840c380/2, rc 0, transno 4294976188 00000100:00100000:2.0:1783520936.060601:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363840c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993207296:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.060618:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f180 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.060652:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503325c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993208192:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.060669:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026281 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x77e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.060671:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026296 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x780:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.060686:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026281 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x77e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.060703:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.060705:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.060707:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.060768:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.060791:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.060792:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002446464 00000100:00100000:1.0:1783520936.060798:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002446464 00000100:00100000:3.0:1783520936.060810:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e3b2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993209216:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.060812:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e3b0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993209088:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.060819:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000091926ded/0xd5401cbabc020f29 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x781:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ad4 expref: 389 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.060847:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002446592 00000100:00100000:1.0:1783520936.060851:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002446592 00000100:00100000:3.0:1783520936.060854:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.060869:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c0e509b3/0xd5401cbabc020f1b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x780:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ac6 expref: 390 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.060876:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36278 00000100:00100000:2.0:1783520936.060880:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.060888:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503325c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+390:2388:x1870153002446464:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.060898:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.060903:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000091926ded/0xd5401cbabc020f29 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x781:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ad4 expref: 390 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.060909:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36279 00000100:00100000:3.0:1783520936.060915:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.060922:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503326680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+390:2388:x1870153002446592:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.060926:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000091926ded/0xd5401cbabc020f29 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x781:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ad4 expref: 390 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.060931:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993209216 00010000:00010000:1.0:1783520936.060934:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.060935:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993209216 00010000:00010000:0.0:1783520936.060941:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x781:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.060943:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c0e509b3/0xd5401cbabc020f1b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x780:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ac6 expref: 390 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.060947:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x781:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.060956:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02628f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x781:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.060960:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363840ed80/2, rc 0, transno 4294976187 00000100:00100000:2.0:1783520936.060962:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993209088 00010000:00010000:1.0:1783520936.060965:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c0e509b3/0xd5401cbabc020f1b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x780:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ac6 expref: 390 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.060967:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993209088 00010000:00010000:0.0:1783520936.060971:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02628f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x781:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.060980:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363840ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993207680:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.060984:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993208576 00000100:00100000:2.0:1783520936.060988:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993208576 00010000:00010000:0.0:1783520936.060998:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.061007:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c0e509b3/0xd5401cbabc020f1b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x780:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ac6 expref: 390 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.061011:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44691 00010000:00010000:1.0:1783520936.061024:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x780:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.061025:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e3b0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+619:3628:x1870152993209216:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.061025:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503325c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+389:2388:x1870153002446464:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 139us (258us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.061030:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x780:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.061032:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:1.0:1783520936.061037:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026296 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x780:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.061040:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1926 count 1 00000100:00100000:3.0:1783520936.061045:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e3b3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993209344:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.061046:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36278 00010000:00010000:2.0:1783520936.061057:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0262ab lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x786:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.061057:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026296 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x780:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.061075:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44692 00010000:00010000:2.0:1783520936.061085:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009f270680/0xd5401cbabc02298c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x786:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f307c expref: 388 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.061085:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.061085:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+619:3627:x1870152993209088:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.061093:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.061101:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x786:0x0].0x0 since=36s v=(0 0) 00002000:00080000:0.0:1783520936.061103:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1925 count 1 00010000:00010000:2.0:1783520936.061105:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x786:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.061109:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0262ab lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x786:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.061114:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0262b2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x785:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.061117:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503326680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+388:2388:x1870153002446592:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 195us (322us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.061132:0:35276:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000091926ded/0xd5401cbabc020f29 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x781:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ad4 expref: 388 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.061134:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36279 00010000:00010000:2.0:1783520936.061135:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009f270680/0xd5401cbabc02298c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x786:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f307c expref: 388 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.061135:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008d6db2ee/0xd5401cbabc020f45 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x785:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b0c expref: 388 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.061154:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e3b3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993208320:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.061154:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x785:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.061158:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009f270680/0xd5401cbabc02298c lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x786:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f307c expref: 386 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.061162:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x785:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.061165:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993209344 00010000:00010000:0.0:1783520936.061170:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0262b2 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x785:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.061171:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993209344 00010000:00010000:3.0:1783520936.061178:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02628f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x781:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.061178:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b6680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993209472:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.061191:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02628f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x781:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.061193:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008d6db2ee/0xd5401cbabc020f45 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x785:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b0c expref: 386 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.061198:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44693 00010000:00010000:3.0:1783520936.061204:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02628f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x781:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.061207:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872180 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.061213:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635784450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+620:3626:x1870152993208576:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.061219:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008d6db2ee/0xd5401cbabc020f45 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x785:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b0c expref: 386 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.061220:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02628f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x781:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.061228:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1923 count 1 00000100:00100000:2.0:1783520936.061229:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44694 00010000:00010000:3.0:1783520936.061239:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02628f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x781:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.061240:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e3b1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+620:3626:x1870152993209344:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.061244:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517332680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993209600:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.061246:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0262b9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x783:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.061265:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b6342a51/0xd5401cbabc020f30 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x783:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1af0 expref: 386 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.061267:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00002000:00080000:2.0:1783520936.061271:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1927 count 1 00010000:00010000:0.0:1783520936.061276:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.061283:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02628f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x781:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.061286:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0262c0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x787:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.061288:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x783:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.061297:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x783:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.061305:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000000166157f/0xd5401cbabc020f53 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x787:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b28 expref: 386 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.061306:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0262b9 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x783:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.061307:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.061312:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02628f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x781:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.061332:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02628f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x781:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.061334:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x787:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.061337:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x787:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.061341:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0262c0 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x787:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.061344:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02628f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x781:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.061351:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b6342a51/0xd5401cbabc020f30 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x783:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1af0 expref: 386 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.061354:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503325880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+621:3628:x1870152993208064:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2425us (2793us total) trans 4294976189 rc 0/0 00000100:00100000:3.0:1783520936.061380:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44687 00010000:00010000:0.0:1783520936.061382:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc02628f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x781:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.061391:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000000166157f/0xd5401cbabc020f53 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x787:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b28 expref: 386 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.061404:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b6342a51/0xd5401cbabc020f30 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x783:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1af0 expref: 386 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.061406:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503327100/2, rc 0, transno 4294976189 00010000:00010000:2.0:1783520936.061409:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000000166157f/0xd5401cbabc020f53 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x787:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b28 expref: 386 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.061412:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.061415:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.061417:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503327100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993208064:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.061419:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.061425:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993209728:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.061438:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b4700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993209856:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.061445:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.061457:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435192190c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.061460:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026296 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x780:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.061469:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872e40 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.061470:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026296 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x780:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.061474:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e3b3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993209984:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.061484:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026296 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x780:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.061513:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026296 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x780:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.061522:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.061530:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520936.061567:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993209984 00000100:00100000:3.0:1783520936.061571:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993209984 00000100:00100000:3.0:1783520936.061593:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44695 00000100:00100000:3.0:1783520936.061603:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e3b0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+622:3626:x1870152993209984:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.061611:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+622:3625:x1870152993207936:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2673us (3032us total) trans 4294976190 rc 0/0 00002000:00080000:3.0:1783520936.061615:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1928 count 1 00000100:00100000:0.0:1783520936.061622:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44688 00010000:00010000:3.0:1783520936.061630:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0262c7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x788:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.061645:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b0c6e952/0xd5401cbabc020f5a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x788:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b36 expref: 386 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.061648:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503327800/2, rc 0, transno 4294976190 00000100:00100000:0.0:1783520936.061660:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503327800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993207936:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.061696:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x788:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.061701:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x788:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.061702:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993210112:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.061710:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0262c7 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x788:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.061732:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.061737:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b0c6e952/0xd5401cbabc020f5a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x788:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b36 expref: 386 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.061749:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b0c6e952/0xd5401cbabc020f5a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x788:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b36 expref: 386 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.061759:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993210112 00000100:00100000:3.0:1783520936.061760:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351dc41500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993210240:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.061765:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993210112 00000100:00100000:2.0:1783520936.061793:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44696 00000100:00100000:3.0:1783520936.061795:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967a80 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.061803:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+622:3627:x1870152993210112:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.061815:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1929 count 1 00010000:00010000:2.0:1783520936.061827:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0262ce lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x789:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.061839:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d6909d71/0xd5401cbabc020f61 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x789:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b44 expref: 386 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.061846:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x789:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.061849:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x789:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.061853:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0262ce lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x789:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.061870:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d6909d71/0xd5401cbabc020f61 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x789:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b44 expref: 386 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.061879:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d6909d71/0xd5401cbabc020f61 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x789:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b44 expref: 386 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.061888:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993210368:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.061909:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872840 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.062056:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.062061:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026296 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x780:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.062069:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.062072:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026296 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x780:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.062080:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026296 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x780:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.062098:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026296 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x780:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.062105:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026296 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x780:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.062120:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.062121:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.062123:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.063108:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e471c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993208832:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.063109:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.063133:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc02629d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x782:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.063141:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503324700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993208960:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.063141:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.063151:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002446720 00000100:00100000:3.0:1783520936.063158:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002446720 00010000:00010000:2.0:1783520936.063170:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0262a4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x784:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.063182:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.063209:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002446848 00000100:00100000:2.0:1783520936.063216:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002446848 00010000:00010000:3.0:1783520936.063230:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e2b92963/0xd5401cbabc020f37 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x782:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ae2 expref: 388 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.063239:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000baf78fb0/0xd5401cbabc020f3e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x784:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1afe expref: 388 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.063276:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002446976 00000100:00100000:2.0:1783520936.063279:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002446976 00010000:00010000:2.0:1783520936.063294:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009f270680/0xd5401cbabc02298c lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x786:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f307c expref: 389 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.063297:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36280 00000100:00100000:3.0:1783520936.063312:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503a69500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+389:2390:x1870153002446848:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.063319:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517332680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993209600:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.063324:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36281 00010000:00010000:3.0:1783520936.063325:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.063335:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000baf78fb0/0xd5401cbabc020f3e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x784:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1afe expref: 389 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.063337:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36282 00000100:00100000:2.0:1783520936.063339:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357e450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+389:2390:x1870153002446720:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.063346:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0262b2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x785:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.063378:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000baf78fb0/0xd5401cbabc020f3e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x784:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1afe expref: 389 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.063379:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.063387:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503a6b800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+389:2390:x1870153002446976:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.063389:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e2b92963/0xd5401cbabc020f37 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x782:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ae2 expref: 389 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.063419:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e2b92963/0xd5401cbabc020f37 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x782:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ae2 expref: 389 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.063422:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.063431:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000baf78fb0/0xd5401cbabc020f3e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x784:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1afe expref: 389 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.063434:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009f270680/0xd5401cbabc02298c lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x786:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f307c expref: 389 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.063442:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e2b92963/0xd5401cbabc020f37 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x782:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ae2 expref: 389 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.063457:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x784:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.063465:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x784:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.063470:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009f270680/0xd5401cbabc02298c lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x786:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f307c expref: 387 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.063472:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0262a4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x784:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.063472:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x782:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.063482:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x782:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.063505:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0262a4 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x784:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.063506:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc02629d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x782:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.063514:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x786:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.063523:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x786:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.063530:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc02629d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x782:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.063533:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0262ab lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x786:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.063538:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.063551:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0262ab lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x786:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.063570:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.063571:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.063576:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503a69500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+387:2390:x1870153002446848:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 268us (437us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.063603:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc02629d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x782:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.063614:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503a6b800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+387:2390:x1870153002446976:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 262us (429us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.063616:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc02629d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x782:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.063620:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357e450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+387:2390:x1870153002446720:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 283us (510us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.063633:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36282 00010000:00010000:1.0:1783520936.063642:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc02629d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x782:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.063648:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36281 00010000:00010000:1.0:1783520936.063658:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc02629d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x782:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.063679:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.063682:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b4700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993209856:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.063696:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.063701:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0262c0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x787:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.063702:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc02629d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x782:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.063726:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.063730:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc02629d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x782:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.063746:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc02629d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x782:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.063750:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36280 00010000:00010000:2.0:1783520936.063767:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc02629d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x782:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.063774:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc02629d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x782:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.063796:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.063799:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.063800:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.063824:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0262a4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x784:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.063830:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e3b2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+623:3628:x1870152993208448:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3541us (3603us total) trans 4294976191 rc 0/0 00010000:00010000:2.0:1783520936.063834:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0262a4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x784:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.063847:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44689 00000004:00080000:0.0:1783520936.063849:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e3b1180/2, rc 0, transno 4294976191 00010000:00010000:2.0:1783520936.063857:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0262a4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x784:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.063866:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0262a4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x784:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.063867:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e3b1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993208448:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.063890:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.063925:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435166cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993210496:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.063928:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.063936:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0262a4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x784:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.063946:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.063950:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0262a4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x784:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.063958:0:35274:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009f270680/0xd5401cbabc02298c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x786:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f307c expref: 384 pid: 20140 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.063962:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0262a4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x784:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.063975:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.063983:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b6680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993209472:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.063990:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0262a4 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x784:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.063999:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.064001:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0262ab lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x786:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.064008:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0262a4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x784:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.064019:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0262ab lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x786:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.064028:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0262ab lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x786:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.064033:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.064037:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.064039:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0262ab lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x786:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.064039:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351dc41500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993210240:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.064041:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.064055:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0262ab lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x786:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.064063:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0262c7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x788:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.064076:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.064078:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.064086:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.064092:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0262ab lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x786:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.064099:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002447232 00000100:00100000:1.0:1783520936.064104:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002447232 00010000:00010000:0.0:1783520936.064110:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.064115:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0262ab lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x786:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.064126:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000000166157f/0xd5401cbabc020f53 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x787:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b28 expref: 384 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.064126:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0262ab lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x786:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.064152:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0262ab lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x786:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.064160:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e3b0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+623:3628:x1870152993209216:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3137us (3306us total) trans 4294976192 rc 0/0 00000100:00100000:1.0:1783520936.064163:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002447104 00010000:00010000:0.0:1783520936.064165:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0262ab lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x786:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.064171:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44691 00000100:00100000:1.0:1783520936.064174:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002447104 00000100:00100000:2.0:1783520936.064186:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993210496 00000100:00100000:2.0:1783520936.064189:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993210496 00010000:00010000:0.0:1783520936.064191:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.064193:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008d6db2ee/0xd5401cbabc020f45 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x785:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b0c expref: 385 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.064195:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.064198:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520936.064219:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e3b2680/2, rc 0, transno 4294976192 00000100:00100000:2.0:1783520936.064225:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44697 00000100:00100000:0.0:1783520936.064233:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e3b2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993209216:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.064236:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503a6bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+623:3628:x1870152993210496:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.064239:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36283 00000100:00100000:1.0:1783520936.064253:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a4e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+385:2387:x1870153002447232:12345-192.168.204.33@tcp:103:ldlm_bl.0 00002000:00080000:2.0:1783520936.064260:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1930 count 1 00010000:00010000:1.0:1783520936.064263:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.064264:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36284 00010000:00010000:1.0:1783520936.064272:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000000166157f/0xd5401cbabc020f53 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x787:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b28 expref: 385 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.064279:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517332680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+385:2387:x1870153002447104:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.064280:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0262d5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x78a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.064282:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1783520936.064296:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435166ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993210624:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.064296:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.064307:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008d6db2ee/0xd5401cbabc020f45 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x785:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b0c expref: 385 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.064311:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003b2122ef/0xd5401cbabc020f68 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x78a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b52 expref: 385 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.064315:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000000166157f/0xd5401cbabc020f53 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x787:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b28 expref: 385 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.064336:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.064337:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x78a:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.064347:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x78a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.064351:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000166157f/0xd5401cbabc020f53 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x787:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b28 expref: 385 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.064355:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0262d5 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x78a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.064394:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x787:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.064396:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008d6db2ee/0xd5401cbabc020f45 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x785:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b0c expref: 385 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.064401:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x787:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.064411:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993210624 00010000:00010000:1.0:1783520936.064412:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0262c0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x787:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.064418:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993210624 00010000:00010000:1.0:1783520936.064428:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0262c0 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x787:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.064429:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008d6db2ee/0xd5401cbabc020f45 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x785:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b0c expref: 384 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.064437:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003b2122ef/0xd5401cbabc020f68 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x78a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b52 expref: 384 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.064450:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44698 00010000:00010000:0.0:1783520936.064457:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x785:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.064461:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x785:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.064464:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435166cbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+624:3626:x1870152993210624:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.064465:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003b2122ef/0xd5401cbabc020f68 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x78a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b52 expref: 383 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.064467:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.064467:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0262b2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x785:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.064500:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1931 count 1 00010000:00010000:0.0:1783520936.064505:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0262b2 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x785:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.064508:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a4e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+383:2387:x1870153002447232:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 255us (512us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.064516:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0262dc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x78b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.064526:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff2d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993210752:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.064532:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36283 00010000:00010000:0.0:1783520936.064543:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.064545:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001b9542c5/0xd5401cbabc020f6f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x78b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b60 expref: 383 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.064553:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x78b:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.064557:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x78b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.064562:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0262dc lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x78b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.064569:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0262b2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x785:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.064579:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872b40 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.064580:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517332680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+382:2387:x1870153002447104:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 304us (502us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.064584:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0262b2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x785:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.064591:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001b9542c5/0xd5401cbabc020f6f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x78b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b60 expref: 382 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.064597:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36284 00010000:00010000:1.0:1783520936.064604:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0262b2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x785:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.064614:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001b9542c5/0xd5401cbabc020f6f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x78b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b60 expref: 381 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.064617:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0262b2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x785:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.064629:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0262c0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x787:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.064634:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435166c8380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993210880:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.064641:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.064647:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0262c0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x787:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.064654:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.064659:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0262c0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x787:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.064661:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0262b2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x785:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.064665:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435129670c0 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.064682:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.064683:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0262c0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x787:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.064688:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0262b2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x785:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.064701:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0262b2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x785:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.064711:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0262b2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x785:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.064713:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.064717:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.064722:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0262b2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x785:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.064724:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0262c0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x787:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.064745:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.064749:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0262c0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x787:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.064753:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.064756:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.064760:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0262c0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x787:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.064760:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.064775:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0262c0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x787:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.064783:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0262c0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x787:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.064804:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.064807:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.064809:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e3b1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+627:3626:x1870152993209344:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3570us (3719us total) trans 4294976194 rc 0/0 00010000:00010000:2.0:1783520936.064811:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.064821:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44694 00000100:00100000:2.0:1783520936.064842:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993210368:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.064857:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+626:3627:x1870152993209088:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3774us (3978us total) trans 4294976195 rc 0/0 00010000:00010000:2.0:1783520936.064862:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0262ce lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x789:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.064863:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44692 00000004:00080000:2.0:1783520936.064897:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e3b0380/2, rc 0, transno 4294976195 00000100:00100000:0.0:1783520936.064898:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000004:00080000:3.0:1783520936.064912:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e3b3b80/2, rc 0, transno 4294976194 00000100:00100000:2.0:1783520936.064914:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e3b0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993209088:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.064922:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e3b3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993209344:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.064927:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002447360 00000100:00100000:1.0:1783520936.064931:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002447360 00010000:00010000:1.0:1783520936.064945:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d6909d71/0xd5401cbabc020f61 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x789:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b44 expref: 382 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.064969:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36285 00000100:00100000:0.0:1783520936.064972:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.064977:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e3b1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+625:3626:x1870152993208704:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4536us (4654us total) trans 4294976193 rc 0/0 00000100:00100000:1.0:1783520936.064984:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503a69880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+382:2387:x1870153002447360:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.064989:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44690 00000004:00080000:3.0:1783520936.064992:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e3b3800/2, rc 0, transno 4294976193 00010000:00010000:1.0:1783520936.065000:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.065001:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e3b3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993208704:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.065010:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d6909d71/0xd5401cbabc020f61 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x789:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b44 expref: 382 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.065019:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002447488 00000100:00100000:0.0:1783520936.065024:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.065026:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002447488 00010000:00010000:1.0:1783520936.065027:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d6909d71/0xd5401cbabc020f61 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x789:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b44 expref: 382 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.065047:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b0c6e952/0xd5401cbabc020f5a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x788:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b36 expref: 383 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.065049:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d6909d71/0xd5401cbabc020f61 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x789:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b44 expref: 383 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.065065:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993209728:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.065076:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x789:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.065079:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x789:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.065082:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435166c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993211264:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.065084:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0262ce lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x789:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.065085:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002447616 00010000:00010000:0.0:1783520936.065089:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0262b9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x783:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.065092:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002447616 00010000:00010000:1.0:1783520936.065102:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0262ce lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x789:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.065110:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b6342a51/0xd5401cbabc020f30 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x783:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1af0 expref: 383 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.065119:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.065131:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.065145:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36286 00000100:00100000:3.0:1783520936.065150:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1783520936.065150:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993211264 00000100:00100000:2.0:1783520936.065157:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503a6ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+383:2390:x1870153002447488:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.065158:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993211264 00000100:00100000:1.0:1783520936.065159:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503a69880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+383:2387:x1870153002447360:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 180us (263us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.065162:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435166c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993211136:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.065173:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.065173:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36285 00010000:00010000:2.0:1783520936.065181:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b0c6e952/0xd5401cbabc020f5a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x788:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b36 expref: 383 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.065191:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36287 00010000:00010000:2.0:1783520936.065195:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b0c6e952/0xd5401cbabc020f5a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x788:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b36 expref: 382 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.065197:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.065201:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44699 00000100:00100000:1.0:1783520936.065203:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357d450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+382:2387:x1870153002447616:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.065216:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.065219:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435166c9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+625:3628:x1870152993211264:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.065226:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435166c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993211008:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.065227:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b6342a51/0xd5401cbabc020f30 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x783:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1af0 expref: 382 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.065237:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b0c6e952/0xd5401cbabc020f5a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x788:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b36 expref: 382 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.065241:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1934 count 1 00010000:00010000:1.0:1783520936.065247:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b6342a51/0xd5401cbabc020f30 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x783:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1af0 expref: 382 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.065253:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.065255:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x788:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.065258:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0262e3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x78e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.065264:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x788:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.065274:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0262c7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x788:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.065277:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b6342a51/0xd5401cbabc020f30 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x783:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1af0 expref: 381 pid: 20140 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.065284:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ee634697/0xd5401cbabc02287b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x78e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f6b expref: 381 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.065286:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993211136 00010000:00010000:2.0:1783520936.065290:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0262c7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x788:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.065291:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993211136 00010000:00010000:1.0:1783520936.065303:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x783:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.065312:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x783:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.065314:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993211008 00000100:00100000:3.0:1783520936.065319:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993211008 00010000:00010000:1.0:1783520936.065320:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0262b9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x783:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.065323:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.065326:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x78e:0x0].0x0 since=36s v=(0 0) 00010000:00010000:1.0:1783520936.065334:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0262b9 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x783:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.065334:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x78e:0x0].0x0 since=0s v=(1 0) 00000100:00100000:3.0:1783520936.065343:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44700 00010000:00010000:0.0:1783520936.065345:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0262e3 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x78e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.065346:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503a6ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+380:2390:x1870153002447488:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 192us (378us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.065380:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435166cb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+627:3628:x1870152993211136:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.065384:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36286 00002000:00080000:3.0:1783520936.065400:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1933 count 1 00010000:00010000:0.0:1783520936.065414:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ee634697/0xd5401cbabc02287b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x78e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f6b expref: 379 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.065415:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0262ea lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x78d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.065421:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.065429:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0262ce lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x789:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.065442:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357d450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+379:2387:x1870153002447616:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 240us (422us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.065442:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ee634697/0xd5401cbabc02287b lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x78e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f6b expref: 379 pid: 9151 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.065445:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000000d34c63a/0xd5401cbabc020f7d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x78d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b7c expref: 379 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.065452:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0262ce lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x789:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.065464:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36287 00010000:00010000:3.0:1783520936.065473:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x78d:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.065477:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0262ce lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x789:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.065478:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x78d:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.065478:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503a6b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993211392:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.065489:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0262ea lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x78d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.065507:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0262ce lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x789:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.065532:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.065533:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000000d34c63a/0xd5401cbabc020f7d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x78d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b7c expref: 378 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.065533:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44701 00000100:00100000:0.0:1783520936.065538:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.065547:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435166c8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+627:3626:x1870152993211008:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.065558:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0262c7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x788:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.065562:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000000d34c63a/0xd5401cbabc020f7d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x78d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b7c expref: 378 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.065563:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1932 count 1 00010000:00010000:0.0:1783520936.065576:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0262c7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x788:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.065578:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0262f1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x78c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.065585:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435166c8a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993211520:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.065600:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0262c7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x788:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.065606:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a90e55b9/0xd5401cbabc020f76 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x78c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b6e expref: 378 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.065617:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0262c7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x788:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.065618:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+628:3627:x1870152993210112:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3816us (3884us total) trans 4294976196 rc 0/0 00010000:00010000:1.0:1783520936.065625:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x78c:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.065635:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x78c:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.065636:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44696 00010000:00010000:1.0:1783520936.065645:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0262f1 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x78c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.065658:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000004:00080000:2.0:1783520936.065666:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b5880/2, rc 0, transno 4294976196 00010000:00010000:0.0:1783520936.065669:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1783520936.065671:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a90e55b9/0xd5401cbabc020f76 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x78c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b6e expref: 378 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.065675:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0262ce lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x789:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.065682:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993210112:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.065689:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a90e55b9/0xd5401cbabc020f76 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x78c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b6e expref: 378 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.065693:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.065698:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.065701:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0262c7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x788:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.065705:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0262ce lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x789:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.065725:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5e00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520936.065732:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517332680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993211648:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.065734:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0262c7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x788:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.065735:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0262ce lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x789:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.065756:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0262c7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x788:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.065764:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607eb8180 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.065764:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0262ce lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x789:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.065771:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0262c7 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x788:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.065777:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0262ce lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x789:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.065783:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0262c7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x788:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.065785:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0262b9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x783:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.065799:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.065804:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.065805:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0262b9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x783:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.065809:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.065812:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0262b9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x783:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.065817:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.065821:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.065823:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0262b9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x783:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.065825:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.065841:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.065853:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.065859:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0262b9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x783:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.065863:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff2d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993210752:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.065871:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:1.0:1783520936.065872:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.065878:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0262b9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x783:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.065883:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435166c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993211776:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.065884:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0262d5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x78a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.065889:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0262b9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x783:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.065902:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e3b0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+628:3626:x1870152993209984:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4298us (4376us total) trans 4294976197 rc 0/0 00000100:00100000:3.0:1783520936.065916:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.065919:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0262b9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x783:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.065920:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44695 00010000:00010000:1.0:1783520936.065931:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0262b9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x783:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.065949:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993211776 00010000:00010000:1.0:1783520936.065952:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.065955:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993211776 00010000:00010000:1.0:1783520936.065956:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520936.065957:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e3b3480/2, rc 0, transno 4294976197 00010000:00010000:1.0:1783520936.065961:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.065966:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e3b3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993209984:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.065985:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44702 00000100:00100000:0.0:1783520936.065994:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435166c8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+628:3626:x1870152993211776:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.066011:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1935 count 1 00010000:00010000:0.0:1783520936.066024:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0262f8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x78f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.066036:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000048af53bc/0xd5401cbabc0213f9 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x78f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b98 expref: 378 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.066043:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635784450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+629:3626:x1870152993208576:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4833us (5130us total) trans 4294976198 rc 0/0 00010000:00010000:0.0:1783520936.066046:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x78f:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.066054:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x78f:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.066059:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44693 00010000:00010000:0.0:1783520936.066062:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0262f8 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x78f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.066088:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000048af53bc/0xd5401cbabc0213f9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x78f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b98 expref: 378 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.066099:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000048af53bc/0xd5401cbabc0213f9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x78f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b98 expref: 378 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.066110:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503a6a300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993212032:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.066135:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1783520936.066147:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993211904:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.066153:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3eb40 going to sleep for 11 seconds 00000004:00080000:1.0:1783520936.066175:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e3b2300/2, rc 0, transno 4294976198 00000100:00100000:3.0:1783520936.066186:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.066189:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e3b2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993208576:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.066208:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993211904 00000100:00100000:2.0:1783520936.066214:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993211904 00000100:00100000:2.0:1783520936.066247:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44703 00000100:00100000:3.0:1783520936.066256:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1783520936.066257:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+629:3625:x1870152993211904:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.066271:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517332300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993212160:0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.066273:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1936 count 1 00000100:00100000:0.0:1783520936.066286:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.066288:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0262ff lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x790:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.066301:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002210ef0e/0xd5401cbabc020f8b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x790:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ba6 expref: 378 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.066307:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002447744 00010000:00010000:2.0:1783520936.066309:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x790:0x0].0x0 since=37s v=(0 0) 00000100:00100000:1.0:1783520936.066313:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002447744 00010000:00010000:2.0:1783520936.066314:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x790:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.066321:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0262ff lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x790:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.066335:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003b2122ef/0xd5401cbabc020f68 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x78a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b52 expref: 379 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.066344:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002210ef0e/0xd5401cbabc020f8b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x790:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ba6 expref: 379 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.066403:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.066418:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002210ef0e/0xd5401cbabc020f8b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x790:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ba6 expref: 379 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.066418:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36288 00000100:00100000:1.0:1783520936.066430:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503a6b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+379:2388:x1870153002447744:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.066436:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff2d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993212288:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.066441:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.066449:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003b2122ef/0xd5401cbabc020f68 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x78a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b52 expref: 379 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.066455:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435166c8380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993210880:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.066474:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003b2122ef/0xd5401cbabc020f68 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x78a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b52 expref: 379 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.066483:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872b40 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.066505:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003b2122ef/0xd5401cbabc020f68 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x78a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b52 expref: 379 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.066514:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993212160 00000100:00100000:2.0:1783520936.066519:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993212160 00010000:00010000:1.0:1783520936.066519:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x78a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.066526:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x78a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.066531:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0262d5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x78a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.066539:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0262d5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x78a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.066551:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44704 00000100:00100000:2.0:1783520936.066562:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+630:3628:x1870152993212160:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.066562:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00002000:00080000:2.0:1783520936.066576:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1937 count 1 00010000:00010000:2.0:1783520936.066588:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026306 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x791:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.066598:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000095118044/0xd5401cbabc020f92 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x791:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bb4 expref: 378 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.066604:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x791:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.066606:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x791:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.066610:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026306 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x791:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.066628:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000095118044/0xd5401cbabc020f92 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x791:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bb4 expref: 378 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.066638:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000095118044/0xd5401cbabc020f92 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x791:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bb4 expref: 378 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.066648:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993212416:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.066668:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503a6b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+378:2388:x1870153002447744:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 240us (382us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.066673:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872480 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.066678:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36288 00010000:00010000:2.0:1783520936.066693:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0262d5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x78a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.066698:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0262dc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x78b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.066705:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0262d5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x78a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.066720:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0262d5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x78a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.066728:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512967a80 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.066728:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0262d5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x78a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.066738:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.066748:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.066798:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.066804:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0262d5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x78a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.066813:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.066817:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0262d5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x78a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.066826:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0262d5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x78a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.066840:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503a6bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+631:3628:x1870152993210496:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2606us (2867us total) trans 4294976199 rc 0/0 00000100:00100000:2.0:1783520936.066847:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44697 00000004:00080000:0.0:1783520936.066852:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435166cb480/2, rc 0, transno 4294976199 00000100:00100000:0.0:1783520936.066867:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435166cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993210496:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.066891:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0262d5 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x78a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.066899:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0262d5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x78a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.066913:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517330380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993212544:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.066917:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.066920:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.066922:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.066934:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002447872 00000100:00100000:3.0:1783520936.066938:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002447872 00000100:00100000:0.0:1783520936.066944:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.066953:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001b9542c5/0xd5401cbabc020f6f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x78b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b60 expref: 378 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.066969:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993212544 00000100:00100000:1.0:1783520936.066974:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993212544 00000100:00100000:3.0:1783520936.066979:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36289 00000100:00100000:3.0:1783520936.066987:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503a68a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+378:2388:x1870153002447872:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.066993:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.066997:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001b9542c5/0xd5401cbabc020f6f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x78b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b60 expref: 378 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.067004:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44705 00010000:00010000:3.0:1783520936.067008:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001b9542c5/0xd5401cbabc020f6f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x78b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b60 expref: 378 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.067013:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503a6b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+631:3628:x1870152993212544:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.067024:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001b9542c5/0xd5401cbabc020f6f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x78b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b60 expref: 378 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.067024:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1938 count 1 00010000:00010000:1.0:1783520936.067035:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02630d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x792:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.067048:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x78b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.067053:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000dff0103e/0xd5401cbabc020fa0 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x792:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bc2 expref: 377 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.067054:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x78b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.067064:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0262dc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x78b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.067064:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x792:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.067071:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x792:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.067076:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02630d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x792:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.067082:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0262dc lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x78b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.067091:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000dff0103e/0xd5401cbabc020fa0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x792:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bc2 expref: 377 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.067097:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000dff0103e/0xd5401cbabc020fa0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x792:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bc2 expref: 377 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.067104:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517331f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993212672:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.067128:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607eb8300 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.067181:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.067214:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0262dc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x78b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.067230:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0262dc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x78b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.067240:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0262dc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x78b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.067243:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503a6b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993211392:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.067255:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0262dc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x78b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.067264:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0262e3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x78e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.067279:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.067305:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:3.0:1783520936.067307:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503a68a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+377:2388:x1870153002447872:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 319us (574us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.067313:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0262dc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x78b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.067318:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36289 00010000:00010000:0.0:1783520936.067327:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.067332:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0262dc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x78b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.067339:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0262dc lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x78b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.067352:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0262dc lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x78b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.067385:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0262dc lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x78b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.067398:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435166cbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+632:3626:x1870152993210624:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2936us (3064us total) trans 4294976200 rc 0/0 00010000:00010000:0.0:1783520936.067402:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520936.067405:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435166ca680/2, rc 0, transno 4294976200 00010000:00010000:0.0:1783520936.067406:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.067407:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44698 00010000:00010000:0.0:1783520936.067410:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.067419:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435166ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993210624:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.067432:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.067457:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002448000 00000100:00100000:2.0:1783520936.067463:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002448000 00010000:00010000:2.0:1783520936.067482:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ee634697/0xd5401cbabc02287b lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x78e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f6b expref: 377 pid: 9151 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.067523:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36290 00000100:00100000:2.0:1783520936.067535:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503a69500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+377:2389:x1870153002448000:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.067544:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.067545:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:2.0:1783520936.067551:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ee634697/0xd5401cbabc02287b lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x78e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f6b expref: 377 pid: 9151 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.067552:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517332680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993211648:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.067556:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993212800:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.067568:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0262f1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x78c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.067573:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ee634697/0xd5401cbabc02287b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x78e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f6b expref: 377 pid: 9151 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.067592:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.067593:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ee634697/0xd5401cbabc02287b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x78e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f6b expref: 377 pid: 9151 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.067605:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x78e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.067609:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x78e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.067614:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0262e3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x78e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.067618:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993212800 00000100:00100000:0.0:1783520936.067624:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993212800 00010000:00010000:2.0:1783520936.067625:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0262e3 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x78e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.067648:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.067659:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0262e3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x78e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.067664:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44706 00010000:00010000:1.0:1783520936.067669:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0262e3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x78e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.067676:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0262e3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x78e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.067677:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+632:3626:x1870152993212800:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.067691:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503a69500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+376:2389:x1870153002448000:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 157us (258us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.067693:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0262e3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x78e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.067696:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1939 count 1 00000100:00100000:2.0:1783520936.067707:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36290 00010000:00010000:0.0:1783520936.067713:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026314 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x793:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.067717:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.067728:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.067729:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f00a497b/0xd5401cbabc020f99 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x793:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bd0 expref: 375 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.067735:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0262e3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x78e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.067742:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x793:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.067751:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x793:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.067752:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbebe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.067760:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0262e3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x78e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.067760:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026314 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x793:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.067782:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0262e3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x78e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.067803:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0262e3 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x78e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.067803:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f00a497b/0xd5401cbabc020f99 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x793:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bd0 expref: 375 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.067810:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435166c9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+633:3628:x1870152993211264:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2596us (2693us total) trans 4294976201 rc 0/0 00000100:00100000:1.0:1783520936.067829:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44699 00010000:00010000:3.0:1783520936.067831:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0262e3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x78e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.067835:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f00a497b/0xd5401cbabc020f99 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x793:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bd0 expref: 375 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.067855:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503a69880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993212928:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.067856:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.067860:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.067862:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520936.067880:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435166c9880/2, rc 0, transno 4294976201 00000100:00100000:0.0:1783520936.067892:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435166c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993211264:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.067941:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.068170:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1783520936.068180:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993213056:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.068209:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.068248:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435166c8a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993211520:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.068267:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993212416:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.068268:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0262ea lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x78d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.068269:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503a6a300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993212032:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.068293:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0262f8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x78f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.068297:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026306 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x791:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.068297:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.068321:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993213056 00000100:00100000:2.0:1783520936.068325:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993213056 00000100:00100000:0.0:1783520936.068329:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002448128 00000100:00100000:0.0:1783520936.068335:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002448128 00000100:00100000:2.0:1783520936.068349:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44707 00000100:00100000:2.0:1783520936.068375:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+633:3625:x1870152993213056:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.068389:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1940 count 1 00010000:00010000:0.0:1783520936.068397:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a90e55b9/0xd5401cbabc020f76 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x78c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b6e expref: 376 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.068402:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02631b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x794:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.068425:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000000695837e/0xd5401cbabc021a20 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x794:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bde expref: 376 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.068432:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36291 00010000:00010000:2.0:1783520936.068436:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x794:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.068441:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x794:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.068443:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503a6ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+376:2388:x1870153002448128:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.068447:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02631b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x794:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.068451:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.068458:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a90e55b9/0xd5401cbabc020f76 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x78c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b6e expref: 376 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.068466:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a90e55b9/0xd5401cbabc020f76 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x78c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b6e expref: 376 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.068475:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000000695837e/0xd5401cbabc021a20 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x794:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bde expref: 376 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.068481:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a90e55b9/0xd5401cbabc020f76 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x78c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b6e expref: 376 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.068502:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000000695837e/0xd5401cbabc021a20 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x794:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bde expref: 376 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.068506:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x78c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.068514:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x78c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.068521:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0262f1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x78c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.068522:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993213184:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.068535:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0262f1 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x78c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.068555:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.068561:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872e40 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.068576:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0262f1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x78c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.068583:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503a6ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+375:2388:x1870153002448128:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 140us (287us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.068606:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.068606:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0262f1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x78c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.068612:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36291 00010000:00010000:1.0:1783520936.068621:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0262f1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x78c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.068630:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0262f1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x78c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.068632:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002448256 00000100:00100000:0.0:1783520936.068638:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002448256 00010000:00010000:1.0:1783520936.068652:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.068656:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000048af53bc/0xd5401cbabc0213f9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x78f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b98 expref: 375 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.068660:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.068668:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0262f1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x78c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.068673:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.068681:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbbe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.068686:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0262f1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x78c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.068687:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002448384 00000100:00100000:0.0:1783520936.068693:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002448384 00010000:00010000:2.0:1783520936.068706:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0262f1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x78c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.068712:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002210ef0e/0xd5401cbabc020f8b lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x790:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ba6 expref: 376 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.068722:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0262f1 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x78c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.068732:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0262f1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x78c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.068740:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36292 00000100:00100000:1.0:1783520936.068748:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435166c8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+634:3626:x1870152993211008:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3203us (3496us total) trans 4294976202 rc 0/0 00010000:00010000:2.0:1783520936.068754:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.068754:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+376:2387:x1870153002448256:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.068760:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.068764:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44701 00000100:00100000:3.0:1783520936.068765:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.068765:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.068770:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.068778:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000048af53bc/0xd5401cbabc0213f9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x78f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b98 expref: 376 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.068789:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000048af53bc/0xd5401cbabc0213f9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x78f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b98 expref: 376 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.068804:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000048af53bc/0xd5401cbabc0213f9 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x78f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b98 expref: 376 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.068812:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435166c9f80/2, rc 0, transno 4294976202 00010000:00010000:0.0:1783520936.068814:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x78f:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.068818:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002448512 00010000:00010000:0.0:1783520936.068819:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x78f:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.068824:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002448512 00000100:00100000:1.0:1783520936.068827:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435166c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993211008:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.068827:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0262f8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x78f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.068840:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0262f8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x78f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.068845:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.068847:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000095118044/0xd5401cbabc020f92 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x791:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bb4 expref: 376 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.068873:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.068883:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002448640 00000100:00100000:3.0:1783520936.068886:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.068889:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002448640 00000100:00100000:0.0:1783520936.068893:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1783520936.068907:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517332680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993213312:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.068908:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000000d34c63a/0xd5401cbabc020f7d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x78d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b7c expref: 377 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.068909:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002448768 00000100:00100000:3.0:1783520936.068916:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002448768 00000100:00100000:2.0:1783520936.068934:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36293 00010000:00010000:3.0:1783520936.068941:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000dff0103e/0xd5401cbabc020fa0 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x792:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bc2 expref: 378 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.068947:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee4a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+378:2388:x1870153002448384:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.068951:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.068963:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.068971:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002210ef0e/0xd5401cbabc020f8b lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x790:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ba6 expref: 378 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.068985:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36294 00010000:00010000:2.0:1783520936.068985:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002210ef0e/0xd5401cbabc020f8b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x790:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ba6 expref: 378 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.068995:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36295 00000100:00100000:3.0:1783520936.068998:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638586850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+378:2387:x1870153002448512:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.069005:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x790:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.069007:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.069011:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee5180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+378:2388:x1870153002448640:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.069013:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x790:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.069017:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000095118044/0xd5401cbabc020f92 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x791:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bb4 expref: 378 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.069023:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0262ff lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x790:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.069028:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.069041:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0262ff lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x790:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.069041:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000000d34c63a/0xd5401cbabc020f7d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x78d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b7c expref: 378 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.069042:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000095118044/0xd5401cbabc020f92 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x791:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bb4 expref: 378 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.069070:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.069077:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000000d34c63a/0xd5401cbabc020f7d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x78d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b7c expref: 378 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.069094:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000d34c63a/0xd5401cbabc020f7d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x78d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1b7c expref: 378 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.069104:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee4a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+378:2388:x1870153002448384:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 160us (434us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.069107:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x78d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.069114:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x78d:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.069116:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36293 00010000:00010000:0.0:1783520936.069122:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0262ea lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x78d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.069136:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36296 00010000:00010000:0.0:1783520936.069138:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0262ea lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x78d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.069148:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee7100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+376:2390:x1870153002448768:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.069154:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000095118044/0xd5401cbabc020f92 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x791:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bb4 expref: 376 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.069164:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x791:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.069165:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.069169:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x791:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.069173:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026306 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x791:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.069173:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.069181:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026306 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x791:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.069182:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000dff0103e/0xd5401cbabc020fa0 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x792:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bc2 expref: 375 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.069194:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee5180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+375:2388:x1870153002448640:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 185us (349us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.069206:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.069206:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000dff0103e/0xd5401cbabc020fa0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x792:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bc2 expref: 375 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.069210:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36295 00010000:00010000:2.0:1783520936.069233:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x792:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.069236:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x792:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.069240:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02630d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x792:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.069247:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02630d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x792:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.069260:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.069262:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+374:2387:x1870153002448256:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 512us (658us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.069271:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36292 00000100:00100000:2.0:1783520936.069287:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee7100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+373:2390:x1870153002448768:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 141us (403us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.069287:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.069292:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638586850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+373:2387:x1870153002448512:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 297us (532us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.069303:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36296 00000100:00100000:3.0:1783520936.069311:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36294 00010000:00010000:0.0:1783520936.069312:0:9151:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000dff0103e/0xd5401cbabc020fa0 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x792:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bc2 expref: 373 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.069327:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002448896 00000100:00100000:0.0:1783520936.069332:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517331f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993212672:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.069333:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002448896 00010000:00010000:2.0:1783520936.069353:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f00a497b/0xd5401cbabc020f99 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x793:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bd0 expref: 371 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.069388:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02630d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x792:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.069394:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0262ea lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x78d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.069395:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026306 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x791:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.069409:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36297 00010000:00010000:3.0:1783520936.069411:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0262ea lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x78d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.069421:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026306 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x791:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.069423:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0262ea lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x78d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.069426:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517333100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+371:2388:x1870153002448896:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.069428:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02630d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x792:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.069439:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.069447:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026306 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x791:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.069448:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f00a497b/0xd5401cbabc020f99 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x793:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bd0 expref: 371 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.069451:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02630d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x792:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.069477:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02630d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x792:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.069479:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f00a497b/0xd5401cbabc020f99 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x793:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bd0 expref: 371 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.069485:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026306 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x791:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.069512:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0262ea lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x78d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.069518:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02630d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x792:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.069527:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x793:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.069535:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x793:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.069539:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520936.069541:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026314 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x793:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.069548:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to regular blp list, 3 blwis in pool 00010000:00010000:2.0:1783520936.069551:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026314 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x793:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.069553:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to regular blp list, 3 blwis in pool 00010000:00010000:3.0:1783520936.069557:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:3.0:1783520936.069563:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026306 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x791:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.069570:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.069570:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520936.069571:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.069574:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026306 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x791:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.069579:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0262ea lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x78d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.069583:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026306 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x791:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.069596:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.069601:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0262ea lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x78d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.069603:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517333100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+371:2388:x1870153002448896:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 183us (319us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.069605:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026306 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x791:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.069625:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36297 00010000:00010000:0.0:1783520936.069627:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0262ea lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x78d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.069628:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+634:3628:x1870152993212160:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3066us (3224us total) trans 4294976203 rc 0/0 00000100:00100000:1.0:1783520936.069648:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44704 00010000:00010000:0.0:1783520936.069658:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0262ea lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x78d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.069662:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0262f8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x78f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.069667:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993213312 00000100:00100000:1.0:1783520936.069672:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993213312 00010000:00010000:0.0:1783520936.069675:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0262ea lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x78d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.069681:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0262f8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x78f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.069696:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0262f8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x78f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.069704:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0262f8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x78f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.069706:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44708 00010000:00010000:0.0:1783520936.069707:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.069715:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02630d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x792:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.069720:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503a6a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+634:3628:x1870152993213312:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.069726:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.069736:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026306 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x791:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.069738:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.069739:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00002000:00080000:1.0:1783520936.069742:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1941 count 1 00010000:00010000:0.0:1783520936.069746:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02630d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x792:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.069748:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0262f8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x78f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.069759:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026322 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x795:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.069767:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.069770:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.069774:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.069775:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.069775:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02630d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x792:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.069782:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.069783:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0262f8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x78f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.069786:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007e77dbb7/0xd5401cbabc021a2e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x795:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bec expref: 370 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.069796:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02630d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x792:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.069806:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0262f8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x78f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.069812:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x795:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.069821:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x795:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.069821:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02630d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x792:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.069829:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0262f8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x78f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.069831:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026322 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x795:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.069849:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.069852:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0262f8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x78f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.069854:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.069860:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.069870:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007e77dbb7/0xd5401cbabc021a2e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x795:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bec expref: 370 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.069876:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.069881:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.069885:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520936.069892:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517332300/2, rc 0, transno 4294976203 00010000:00010000:1.0:1783520936.069893:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007e77dbb7/0xd5401cbabc021a2e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x795:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bec expref: 370 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.069909:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517332a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993213440:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.069909:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517332300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993212160:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.069937:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435166cb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+636:3628:x1870152993211136:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4583us (4744us total) trans 4294976204 rc 0/0 00000100:00100000:3.0:1783520936.069943:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44700 00000100:00100000:1.0:1783520936.069960:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607eb8a80 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.069971:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1783520936.069984:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503a6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993213568:0@lo:6:osp-syn-1-0.0 00000004:00080000:0.0:1783520936.069986:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435166c9180/2, rc 0, transno 4294976204 00000100:00100000:0.0:1783520936.069999:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435166c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993211136:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.070018:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.070040:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993213184:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.070055:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02631b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x794:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.070064:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993213568 00000100:00100000:0.0:1783520936.070067:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503a6b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993213696:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.070069:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993213568 00000100:00100000:2.0:1783520936.070100:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44709 00000100:00100000:2.0:1783520936.070107:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+636:3627:x1870152993213568:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.070113:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00080000:2.0:1783520936.070119:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1942 count 1 00000100:00100000:1.0:1783520936.070124:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.070131:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026329 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x796:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.070142:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000047715579/0xd5401cbabc022ad5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x796:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f31c5 expref: 370 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.070147:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x796:0x0].0x0 since=36s v=(0 0) 00000100:00100000:3.0:1783520936.070148:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002449024 00010000:00010000:2.0:1783520936.070150:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x796:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.070154:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026329 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x796:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.070155:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002449024 00010000:00010000:0.0:1783520936.070167:0:35282:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002210ef0e/0xd5401cbabc020f8b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x790:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ba6 expref: 370 pid: 20415 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.070172:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000047715579/0xd5401cbabc022ad5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x796:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31c5 expref: 370 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.070178:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993213696 00000100:00100000:1.0:1783520936.070185:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993213696 00010000:00010000:2.0:1783520936.070194:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000047715579/0xd5401cbabc022ad5 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x796:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31c5 expref: 370 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.070201:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff2d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993212288:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.070214:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993213824:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.070220:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44710 00010000:00010000:0.0:1783520936.070222:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0262ff lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x790:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.070228:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000000695837e/0xd5401cbabc021a20 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x794:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bde expref: 370 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.070232:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503a68a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+637:3625:x1870152993213696:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.070237:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0262ff lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x790:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.070251:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872480 going to sleep for 11 seconds 00002000:00080000:1.0:1783520936.070251:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1943 count 1 00010000:00010000:0.0:1783520936.070254:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0262ff lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x790:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.070265:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0262ff lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x790:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.070269:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026330 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x797:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.070273:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36298 00010000:00010000:0.0:1783520936.070279:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0262ff lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x790:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.070285:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517333480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+370:2389:x1870153002449024:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.070291:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008224ace0/0xd5401cbabc021a3c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x797:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c08 expref: 370 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.070297:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.070305:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000000695837e/0xd5401cbabc021a20 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x794:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bde expref: 370 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.070307:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x797:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.070312:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.070314:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x797:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.070324:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026330 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x797:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.070331:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000000695837e/0xd5401cbabc021a20 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x794:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bde expref: 370 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.070341:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435166c8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+637:3626:x1870152993211776:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4347us (4424us total) trans 4294976205 rc 0/0 00010000:00010000:3.0:1783520936.070350:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000695837e/0xd5401cbabc021a20 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x794:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bde expref: 370 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.070353:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44702 00010000:00010000:1.0:1783520936.070381:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008224ace0/0xd5401cbabc021a3c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x797:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c08 expref: 370 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.070400:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008224ace0/0xd5401cbabc021a3c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x797:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c08 expref: 370 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.070413:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517332d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993213952:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.070431:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x794:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.070433:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x794:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.070437:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02631b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x794:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.070444:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02631b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x794:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.070444:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435166c9500/2, rc 0, transno 4294976205 00000100:00100000:0.0:1783520936.070451:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+637:3625:x1870152993211904:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4195us (4262us total) trans 4294976206 rc 0/0 00010000:00010000:3.0:1783520936.070461:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520936.070462:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44703 00000100:00100000:2.0:1783520936.070465:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435166c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993211776:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.070467:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607eb8180 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.070468:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0262ff lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x790:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.070477:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.070481:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0262ff lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x790:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.070489:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0262ff lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x790:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.070513:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943508ee4700/2, rc 0, transno 4294976206 00010000:00010000:3.0:1783520936.070517:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0262ff lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x790:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.070522:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993211904:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.070532:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:3.0:1783520936.070533:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0262ff lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x790:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.070544:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514ff2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993214080:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.070554:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.070558:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.070562:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.070566:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1783520936.070578:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.070581:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503325880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993214208:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.070608:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993214080 00000100:00100000:2.0:1783520936.070614:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993214080 00000100:00100000:1.0:1783520936.070620:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.070625:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.070650:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993214208 00000100:00100000:2.0:1783520936.070654:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993214208 00000100:00100000:1.0:1783520936.070660:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44711 00000100:00100000:1.0:1783520936.070671:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503a69500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+639:3625:x1870152993214080:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.070683:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44712 00002000:00080000:1.0:1783520936.070684:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1944 count 1 00000100:00100000:2.0:1783520936.070694:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517333b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+639:3628:x1870152993214208:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.070703:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517333480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+369:2389:x1870153002449024:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 415us (577us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.070704:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503a6b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+639:3628:x1870152993212544:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3691us (3759us total) trans 4294976207 rc 0/0 00002000:00080000:2.0:1783520936.070712:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1945 count 1 00000100:00100000:3.0:1783520936.070713:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36298 00000004:00080000:1.0:1783520936.070716:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517330380/2, rc 0, transno 4294976207 00000100:00100000:0.0:1783520936.070718:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44705 00010000:00010000:2.0:1783520936.070730:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026337 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x799:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.070731:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517330380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993212544:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.070751:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000dbdfc66f/0xd5401cbabc021a51 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x799:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c24 expref: 368 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.070753:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02631b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x794:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.070766:0:178496:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f00a497b/0xd5401cbabc020f99 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x793:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bd0 expref: 368 pid: 9151 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.070767:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x799:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.070773:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02631b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x794:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.070776:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x799:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.070786:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517331f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993214336:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.070787:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026337 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x799:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.070796:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503a69880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993212928:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.070800:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02631b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x794:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.070818:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026314 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x793:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.070828:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000dbdfc66f/0xd5401cbabc021a51 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x799:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c24 expref: 367 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.070828:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.070836:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026314 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x793:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.070841:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000dbdfc66f/0xd5401cbabc021a51 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x799:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c24 expref: 367 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.070855:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026314 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x793:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.070861:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503325c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993214464:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.070874:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc02633e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x798:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.070875:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02631b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x794:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.070875:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026314 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x793:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.070888:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872180 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.070904:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026314 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x793:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.070909:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c0cdb672/0xd5401cbabc021a4a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x798:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c16 expref: 367 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.070924:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.070932:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x798:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.070934:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520936.070935:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1783520936.070936:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x798:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.070942:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02631b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x794:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.070944:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc02633e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x798:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.070946:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.070955:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026314 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x793:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.070971:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.070976:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.070977:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02631b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x794:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.070985:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026314 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x793:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.070987:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02631b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x794:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.070989:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c0cdb672/0xd5401cbabc021a4a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x798:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c16 expref: 367 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.071012:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026314 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x793:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.071019:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02631b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x794:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.071021:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+639:3625:x1870152993213056:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2661us (2806us total) trans 4294976208 rc 0/0 00010000:00010000:1.0:1783520936.071021:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c0cdb672/0xd5401cbabc021a4a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x798:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c16 expref: 367 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.071031:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026314 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x793:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.071041:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44707 00010000:00010000:2.0:1783520936.071047:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02631b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x794:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.071054:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515776680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993214592:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.071054:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026314 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x793:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.071072:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.071077:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520936.071079:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943508ee6300/2, rc 0, transno 4294976208 00010000:00010000:2.0:1783520936.071080:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.071082:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.071087:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.071089:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607eb8300 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.071090:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.071098:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993213056:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.071104:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993214336 00000100:00100000:0.0:1783520936.071110:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993214336 00000100:00100000:0.0:1783520936.071143:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44713 00000100:00100000:2.0:1783520936.071150:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1783520936.071154:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634573b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+639:3628:x1870152993214336:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.071163:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503326680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993214720:0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.071168:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1946 count 1 00010000:00010000:0.0:1783520936.071183:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026345 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x79a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.071194:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000091f9ed30/0xd5401cbabc021a5f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x79a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c32 expref: 367 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.071203:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.071209:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x79a:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.071213:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x79a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.071217:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026345 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x79a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.071239:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000091f9ed30/0xd5401cbabc021a5f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x79a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c32 expref: 367 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.071249:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000091f9ed30/0xd5401cbabc021a5f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x79a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c32 expref: 367 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.071253:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517332a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993213440:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.071267:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503a69880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993214848:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.071268:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026322 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x795:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.071304:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.071324:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993214720 00000100:00100000:0.0:1783520936.071328:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993214720 00000100:00100000:3.0:1783520936.071380:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44714 00000100:00100000:3.0:1783520936.071389:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503324700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+640:3626:x1870152993214720:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520936.071400:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1947 count 1 00010000:00010000:3.0:1783520936.071411:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc02634c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x79b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.071451:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003210abbd/0xd5401cbabc021a58 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x79b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c40 expref: 367 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.071456:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x79b:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.071459:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x79b:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.071461:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+641:3626:x1870152993212800:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3787us (3871us total) trans 4294976209 rc 0/0 00010000:00010000:3.0:1783520936.071463:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc02634c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x79b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.071474:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44706 00010000:00010000:3.0:1783520936.071487:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003210abbd/0xd5401cbabc021a58 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x79b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c40 expref: 367 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.071507:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003210abbd/0xd5401cbabc021a58 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x79b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c40 expref: 367 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.071515:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993214976:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.071539:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435129670c0 going to sleep for 11 seconds 00000004:00080000:2.0:1783520936.071708:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943508ee5c00/2, rc 0, transno 4294976209 00000100:00100000:2.0:1783520936.071724:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993212800:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.071775:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993213824:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.071790:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517332a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993215104:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.071791:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026329 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x796:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.071825:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.071849:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993215104 00000100:00100000:1.0:1783520936.071853:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993215104 00000100:00100000:2.0:1783520936.071869:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.071878:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44715 00000100:00100000:1.0:1783520936.071884:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363840df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+641:3626:x1870152993215104:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.071896:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1948 count 1 00010000:00010000:1.0:1783520936.071906:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026353 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x79c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.071909:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.071913:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002449152 00000100:00100000:0.0:1783520936.071920:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002449152 00010000:00010000:1.0:1783520936.071923:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006a33a79f/0xd5401cbabc021a66 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x79c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c4e expref: 367 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.071934:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x79c:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.071936:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x79c:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.071939:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002449280 00010000:00010000:1.0:1783520936.071939:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026353 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x79c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.071945:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007e77dbb7/0xd5401cbabc021a2e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x795:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bec expref: 368 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.071946:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002449280 00010000:00010000:3.0:1783520936.071970:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000047715579/0xd5401cbabc022ad5 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x796:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31c5 expref: 369 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.071970:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006a33a79f/0xd5401cbabc021a66 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x79c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c4e expref: 369 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.071979:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36299 00000100:00100000:0.0:1783520936.071995:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363840d880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+369:2388:x1870153002449152:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.072002:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36300 00010000:00010000:1.0:1783520936.072002:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006a33a79f/0xd5401cbabc021a66 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x79c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c4e expref: 369 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.072009:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.072018:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007e77dbb7/0xd5401cbabc021a2e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x795:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bec expref: 369 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.072022:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e3b1500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993215232:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.072027:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363840e680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+369:2388:x1870153002449280:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.072042:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.072042:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007e77dbb7/0xd5401cbabc021a2e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x795:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bec expref: 369 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.072053:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000047715579/0xd5401cbabc022ad5 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x796:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31c5 expref: 369 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.072064:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007e77dbb7/0xd5401cbabc021a2e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x795:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1bec expref: 369 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.072069:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000047715579/0xd5401cbabc022ad5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x796:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31c5 expref: 369 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.072076:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607eb8a80 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.072086:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x795:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.072091:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x795:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.072094:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000047715579/0xd5401cbabc022ad5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x796:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31c5 expref: 368 pid: 35274 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.072099:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026322 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x795:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.072106:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x796:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.072110:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026322 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x795:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.072111:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x796:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.072122:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026329 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x796:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.072132:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026329 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x796:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.072139:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.072157:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.072157:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026322 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x795:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.072168:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026322 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x795:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.072172:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363840d880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+367:2388:x1870153002449152:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 180us (302us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.072187:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363840e680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+367:2388:x1870153002449280:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 172us (277us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.072187:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026322 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x795:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.072187:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36299 00000100:00100000:2.0:1783520936.072203:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36300 00010000:00010000:3.0:1783520936.072210:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026329 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x796:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.072212:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026322 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x795:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.072222:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026329 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x796:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.072257:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.072309:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.072314:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026322 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x795:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.072318:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026329 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x796:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.072321:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.072327:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026322 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x795:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.072337:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026329 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x796:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.072341:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026322 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x795:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.072352:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503a6a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+642:3628:x1870152993213312:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2634us (3403us total) trans 4294976210 rc 0/0 00010000:00010000:2.0:1783520936.072384:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026322 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x795:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.072396:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44708 00010000:00010000:2.0:1783520936.072398:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026322 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x795:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.072417:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.072429:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.072430:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.072434:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026329 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x796:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.072441:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.072445:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026329 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x796:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.072453:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026329 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x796:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.072464:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026329 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x796:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.072472:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026329 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x796:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.072489:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.072505:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.072508:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520936.072602:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517332680/2, rc 0, transno 4294976210 00000100:00100000:0.0:1783520936.072616:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517332680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993213312:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.072620:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517332d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993213952:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.072640:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026330 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x797:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.072675:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.072682:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1783520936.072690:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+642:3627:x1870152993213568:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2582us (2671us total) trans 4294976211 rc 0/0 00000100:00100000:0.0:1783520936.072696:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503a6ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993215360:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.072702:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44709 00000100:00100000:2.0:1783520936.072703:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002449408 00000100:00100000:2.0:1783520936.072709:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002449408 00000004:00080000:1.0:1783520936.072712:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503a6b480/2, rc 0, transno 4294976211 00000100:00100000:1.0:1783520936.072727:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503a6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993213568:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.072733:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008224ace0/0xd5401cbabc021a3c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x797:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c08 expref: 366 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.072737:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.072764:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36301 00000100:00100000:3.0:1783520936.072769:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993215360 00000100:00100000:2.0:1783520936.072774:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638585050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+366:2388:x1870153002449408:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.072775:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e3b3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993215488:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.072776:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993215360 00010000:00010000:2.0:1783520936.072788:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.072795:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008224ace0/0xd5401cbabc021a3c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x797:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c08 expref: 366 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.072810:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008224ace0/0xd5401cbabc021a3c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x797:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c08 expref: 366 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.072817:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44716 00000100:00100000:0.0:1783520936.072823:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.072827:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.072833:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+642:3628:x1870152993215360:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.072835:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008224ace0/0xd5401cbabc021a3c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x797:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c08 expref: 366 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.072851:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1949 count 1 00010000:00010000:2.0:1783520936.072852:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x797:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.072859:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x797:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.072861:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993215488 00010000:00010000:2.0:1783520936.072864:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026330 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x797:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.072866:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993215488 00010000:00010000:3.0:1783520936.072868:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc02635a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x79d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.072887:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026330 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x797:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.072889:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.072894:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44717 00010000:00010000:3.0:1783520936.072896:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f8d30a62/0xd5401cbabc021a74 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x79d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c5c expref: 365 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.072905:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+643:3628:x1870152993215488:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.072917:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x79d:0x0].0x0 since=37s v=(0 0) 00000100:00100000:1.0:1783520936.072917:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002449536 00002000:00080000:0.0:1783520936.072921:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1950 count 1 00010000:00010000:2.0:1783520936.072922:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.072923:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x79d:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.072923:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002449536 00010000:00010000:3.0:1783520936.072930:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc02635a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x79d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.072937:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026361 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x79e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.072947:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638585050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+366:2388:x1870153002449408:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 176us (272us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.072947:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000dbdfc66f/0xd5401cbabc021a51 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x799:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c24 expref: 366 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.072956:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f8d30a62/0xd5401cbabc021a74 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x79d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c5c expref: 366 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.072957:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e56f9fb2/0xd5401cbabc0229c4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x79e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f30b4 expref: 366 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.072962:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36301 00010000:00010000:3.0:1783520936.072982:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f8d30a62/0xd5401cbabc021a74 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x79d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c5c expref: 365 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.072982:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x79e:0x0].0x0 since=36s v=(0 0) 00000100:00100000:1.0:1783520936.072990:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002449664 00010000:00010000:0.0:1783520936.072991:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x79e:0x0].0x0 since=0s v=(1 0) 00000100:00100000:1.0:1783520936.072995:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002449664 00000100:00100000:3.0:1783520936.073001:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993215616:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.073001:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026361 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x79e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.073015:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c0cdb672/0xd5401cbabc021a4a lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x798:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c16 expref: 366 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.073031:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e56f9fb2/0xd5401cbabc0229c4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x79e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30b4 expref: 366 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.073042:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e56f9fb2/0xd5401cbabc0229c4 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x79e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30b4 expref: 366 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.073043:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36302 00000100:00100000:0.0:1783520936.073056:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a5500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993215744:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.073057:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e3b2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+366:2390:x1870153002449536:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.073070:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.073075:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c016cc0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.073077:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36303 00010000:00010000:1.0:1783520936.073078:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000dbdfc66f/0xd5401cbabc021a51 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x799:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c24 expref: 366 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.073091:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e3b0a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+366:2389:x1870153002449664:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.073094:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000dbdfc66f/0xd5401cbabc021a51 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x799:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c24 expref: 366 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.073096:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.073105:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.073113:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c0cdb672/0xd5401cbabc021a4a lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x798:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c16 expref: 366 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.073115:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x799:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.073122:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x799:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.073129:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026337 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x799:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.073137:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c0cdb672/0xd5401cbabc021a4a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x798:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c16 expref: 366 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.073140:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026337 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x799:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.073162:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x798:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.073165:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x798:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.073167:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.073169:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc02633e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x798:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.073175:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc02633e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x798:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.073187:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.073201:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e3b2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+366:2390:x1870153002449536:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 146us (379us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.073212:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e3b0a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+366:2389:x1870153002449664:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 122us (324us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.073212:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36302 00000100:00100000:2.0:1783520936.073224:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36303 00010000:00010000:1.0:1783520936.073254:0:20415:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c0cdb672/0xd5401cbabc021a4a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x798:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c16 expref: 364 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.073255:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026330 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x797:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.073272:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026330 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x797:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.073280:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026330 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x797:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.073282:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515776680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993214592:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.073287:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026330 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x797:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.073299:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.073302:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc02633e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x798:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.073307:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.073312:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026330 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x797:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.073328:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.073332:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026330 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x797:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.073334:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc02633e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x798:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.073354:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026330 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x797:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.073391:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026330 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x797:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.073391:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc02633e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x798:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.073407:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026330 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x797:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.073413:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc02633e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x798:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.073420:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.073422:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc02633e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x798:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.073423:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.073427:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.073443:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.073452:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.073459:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc02633e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x798:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.073468:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.073471:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc02633e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x798:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.073478:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc02633e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x798:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.073487:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc02633e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x798:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.073505:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc02633e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x798:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.073533:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.073535:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.073539:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.073540:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503a68a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+645:3625:x1870152993213696:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3311us (3429us total) trans 4294976212 rc 0/0 00000100:00100000:2.0:1783520936.073549:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44710 00000100:00100000:1.0:1783520936.073554:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503a69500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+645:3625:x1870152993214080:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2883us (2974us total) trans 4294976213 rc 0/0 00000100:00100000:1.0:1783520936.073563:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44711 00000004:00080000:3.0:1783520936.073592:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514ff2d80/2, rc 0, transno 4294976213 00010000:00010000:2.0:1783520936.073593:0:35275:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000dbdfc66f/0xd5401cbabc021a51 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x799:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c24 expref: 363 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.073606:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514ff2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993214080:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.073608:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503325c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993214464:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.073630:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026337 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x799:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.073632:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503a6b100/2, rc 0, transno 4294976212 00000100:00100000:0.0:1783520936.073639:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.073644:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503a6b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993213696:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.073655:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1783520936.073664:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e3b1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993215872:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.073675:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.073680:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993214976:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.073700:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc02634c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x79b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.073713:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.073713:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e3b1500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993215232:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.073716:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1783520936.073721:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.073729:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515776680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993216000:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.073738:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026353 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x79c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.073753:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002449792 00000100:00100000:0.0:1783520936.073759:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002449792 00000100:00100000:2.0:1783520936.073761:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002450048 00000100:00100000:3.0:1783520936.073766:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.073767:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002450048 00000100:00100000:1.0:1783520936.073771:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002449920 00000100:00100000:1.0:1783520936.073777:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002449920 00010000:00010000:0.0:1783520936.073781:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003210abbd/0xd5401cbabc021a58 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x79b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c40 expref: 364 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.073789:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006a33a79f/0xd5401cbabc021a66 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x79c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c4e expref: 365 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.073799:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000091f9ed30/0xd5401cbabc021a5f lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x79a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c32 expref: 365 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.073839:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36304 00000100:00100000:0.0:1783520936.073850:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36305 00000100:00100000:2.0:1783520936.073854:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+365:2390:x1870153002450048:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520936.073863:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36306 00000100:00100000:0.0:1783520936.073865:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a7b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+365:2390:x1870153002449792:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.073867:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.073878:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006a33a79f/0xd5401cbabc021a66 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x79c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c4e expref: 365 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.073879:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993215872 00010000:00010000:0.0:1783520936.073879:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.073885:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993215872 00010000:00010000:0.0:1783520936.073890:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003210abbd/0xd5401cbabc021a58 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x79b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c40 expref: 365 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.073897:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006a33a79f/0xd5401cbabc021a66 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x79c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c4e expref: 365 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.073905:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003210abbd/0xd5401cbabc021a58 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x79b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c40 expref: 365 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.073906:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993216000 00000100:00100000:1.0:1783520936.073912:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993216000 00010000:00010000:2.0:1783520936.073919:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006a33a79f/0xd5401cbabc021a66 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x79c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c4e expref: 365 pid: 35274 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.073924:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003210abbd/0xd5401cbabc021a58 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x79b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c40 expref: 365 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.073934:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44718 00010000:00010000:2.0:1783520936.073938:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x79c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.073944:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x79c:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.073949:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363840d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+645:3626:x1870152993215872:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.073949:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x79b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.073952:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026353 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.073959:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x79b:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.073966:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a4e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+363:2390:x1870153002449920:12345-192.168.204.33@tcp:103:ldlm_bl.0 00002000:00080000:1.0:1783520936.073969:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1951 count 1 00010000:00010000:2.0:1783520936.073970:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026353 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x79c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.073971:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc02634c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.073976:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.073987:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026368 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x79f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.073992:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000091f9ed30/0xd5401cbabc021a5f lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x79a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c32 expref: 363 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.074001:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc02634c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x79b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.074022:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005d902a4d/0xd5401cbabc021a82 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x79f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c78 expref: 363 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.074023:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.074027:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000091f9ed30/0xd5401cbabc021a5f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x79a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c32 expref: 363 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.074040:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x79f:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.074040:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.074046:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x79f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.074050:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x79a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.074052:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026368 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x79f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.074058:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x79a:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.074058:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+363:2390:x1870153002450048:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 207us (385us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.074070:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a7b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+363:2390:x1870153002449792:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 206us (433us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.074075:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36304 00000100:00100000:0.0:1783520936.074085:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36305 00010000:00010000:1.0:1783520936.074088:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005d902a4d/0xd5401cbabc021a82 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x79f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c78 expref: 362 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.074098:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026337 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x799:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.074102:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005d902a4d/0xd5401cbabc021a82 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x79f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c78 expref: 361 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.074111:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026337 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x799:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.074112:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026345 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.074115:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc02634c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.074130:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e3b1500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993216128:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.074136:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026337 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x799:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.074138:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026345 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x79a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.074145:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc02634c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.074146:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026337 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x799:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.074168:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc02634c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.074172:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435190663c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.074173:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.074185:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc02634c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x79b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.074189:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026353 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.074192:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.074200:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026337 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x799:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.074207:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026353 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.074211:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.074214:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.074218:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026337 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x799:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.074218:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.074227:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc02634c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x79b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.074228:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026353 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.074231:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026337 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x799:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.074251:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.074255:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026353 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x79c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.074259:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc02634c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x79b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.074261:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026337 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x799:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.074283:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026337 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x799:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.074283:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.074297:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc02634c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x79b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.074301:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503324700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+647:3626:x1870152993214720:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2912us (3101us total) trans 4294976215 rc 0/0 00010000:00010000:3.0:1783520936.074306:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.074313:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026353 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x79c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.074319:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44714 00010000:00010000:2.0:1783520936.074322:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc02634c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x79b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.074326:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.074332:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026353 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x79c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.074335:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc02634c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x79b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.074342:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026353 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x79c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.074343:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44719 00000100:00100000:0.0:1783520936.074384:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+647:3625:x1870152993216000:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.074398:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026353 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x79c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.074398:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1952 count 1 00010000:00010000:2.0:1783520936.074401:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.074404:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363840df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+647:3626:x1870152993215104:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2516us (2575us total) trans 4294976216 rc 0/0 00010000:00010000:2.0:1783520936.074408:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.074413:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.074414:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02636f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7a0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.074418:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.074419:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026353 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x79c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.074424:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44715 00010000:00010000:0.0:1783520936.074436:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000723b120a/0xd5401cbabc021a7b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c86 expref: 361 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.074441:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517332a00/2, rc 0, transno 4294976216 00010000:00010000:0.0:1783520936.074442:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a0:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.074448:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.074449:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.074454:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.074455:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02636f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.074460:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.074463:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517332a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993215104:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.074470:0:35282:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000091f9ed30/0xd5401cbabc021a5f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x79a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c32 expref: 361 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.074473:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000723b120a/0xd5401cbabc021a7b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c86 expref: 361 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.074478:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000004:00080000:2.0:1783520936.074483:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503326680/2, rc 0, transno 4294976215 00000100:00100000:2.0:1783520936.074502:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503326680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993214720:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.074508:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000723b120a/0xd5401cbabc021a7b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c86 expref: 361 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.074516:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503a69880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993214848:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.074518:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a4e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+360:2390:x1870153002449920:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 642us (799us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.074522:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993216256:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.074536:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36306 00010000:00010000:1.0:1783520936.074539:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026345 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.074564:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3eb40 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.074583:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517333b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+646:3628:x1870152993214208:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3892us (3965us total) trans 4294976214 rc 0/0 00000100:00100000:2.0:1783520936.074592:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44712 00000100:00100000:1.0:1783520936.074606:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503a69880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993216384:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.074621:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993215616:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.074622:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517333100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993216512:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.074647:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc02635a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x79d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.074648:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.074669:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993216384 00000100:00100000:2.0:1783520936.074673:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993216384 00000100:00100000:0.0:1783520936.074692:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a5500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993215744:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.074702:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.074709:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44720 00000100:00100000:1.0:1783520936.074710:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.074713:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026361 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x79e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.074722:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517333480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+646:3627:x1870152993216384:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.074738:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1953 count 1 00000004:00080000:1.0:1783520936.074748:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503325880/2, rc 0, transno 4294976214 00010000:00010000:2.0:1783520936.074751:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026376 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7a1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.074762:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503325880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993214208:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.074766:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000021ab238e/0xd5401cbabc021a89 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c94 expref: 359 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.074775:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a1:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.074779:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a1:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.074782:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.074785:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993216512 00010000:00010000:2.0:1783520936.074787:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026376 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.074792:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993216512 00010000:00010000:2.0:1783520936.074817:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000021ab238e/0xd5401cbabc021a89 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c94 expref: 359 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.074822:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993216640:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.074827:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44721 00010000:00010000:2.0:1783520936.074834:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000021ab238e/0xd5401cbabc021a89 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c94 expref: 359 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.074841:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517332d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+647:3628:x1870152993216512:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.074849:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363840e680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993216768:192.168.204.33@tcp:104: 00002000:00080000:1.0:1783520936.074856:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1954 count 1 00010000:00010000:1.0:1783520936.074869:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02637d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7a2:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.074873:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002450176 00000100:00100000:2.0:1783520936.074878:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002450176 00010000:00010000:1.0:1783520936.074885:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000096f73924/0xd5401cbabc021a90 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ca2 expref: 360 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.074894:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a2:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.074897:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.074900:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f8d30a62/0xd5401cbabc021a74 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x79d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c5c expref: 360 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.074904:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02637d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.074907:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.074935:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002450304 00010000:00010000:1.0:1783520936.074938:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000096f73924/0xd5401cbabc021a90 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ca2 expref: 360 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.074940:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002450304 00000100:00100000:0.0:1783520936.074949:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993216640 00010000:00010000:1.0:1783520936.074954:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000096f73924/0xd5401cbabc021a90 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ca2 expref: 361 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.074955:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993216640 00010000:00010000:2.0:1783520936.074960:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e56f9fb2/0xd5401cbabc0229c4 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x79e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30b4 expref: 361 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.074986:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44722 00000100:00100000:1.0:1783520936.074988:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e471f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993216896:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.075000:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36307 00000100:00100000:0.0:1783520936.075000:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635785050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+648:3625:x1870152993216640:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.075012:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee7b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+361:2387:x1870153002450176:12345-192.168.204.33@tcp:103:ldlm_bl.0 00002000:00080000:0.0:1783520936.075016:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1955 count 1 00010000:00010000:2.0:1783520936.075022:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.075022:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff94363865b240 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.075032:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f8d30a62/0xd5401cbabc021a74 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x79d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c5c expref: 361 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.075033:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026384 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7a3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.075044:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36308 00010000:00010000:2.0:1783520936.075056:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f8d30a62/0xd5401cbabc021a74 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x79d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c5c expref: 361 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.075060:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638584450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+361:2387:x1870153002450304:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.075060:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000946c731f/0xd5401cbabc021a97 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cb0 expref: 361 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.075073:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a3:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.075076:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.075077:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.075081:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f8d30a62/0xd5401cbabc021a74 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x79d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c5c expref: 361 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.075083:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026384 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.075086:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e56f9fb2/0xd5401cbabc0229c4 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x79e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30b4 expref: 361 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.075106:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x79d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.075111:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x79d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.075115:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e56f9fb2/0xd5401cbabc0229c4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x79e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30b4 expref: 360 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.075116:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc02635a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.075121:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000946c731f/0xd5401cbabc021a97 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cb0 expref: 360 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.075142:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc02635a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x79d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.075144:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000946c731f/0xd5401cbabc021a97 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cb0 expref: 360 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.075147:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e56f9fb2/0xd5401cbabc0229c4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x79e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30b4 expref: 360 pid: 20415 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.075171:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351dc40700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993217024:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.075173:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x79e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.075177:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x79e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.075182:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026361 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.075184:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.075193:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026361 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x79e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.075196:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.075212:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee7b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+359:2387:x1870153002450176:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 201us (510us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.075213:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc02635a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.075218:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.075224:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026361 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.075237:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc02635a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.075237:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36307 00000100:00100000:1.0:1783520936.075244:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638584450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+359:2387:x1870153002450304:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 187us (460us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.075248:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026361 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.075261:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36308 00010000:00010000:3.0:1783520936.075262:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc02635a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.075271:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026361 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.075275:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc02635a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x79d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.075285:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872480 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.075302:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026361 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x79e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.075313:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.075313:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026345 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.075322:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.075324:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026345 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.075327:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc02635a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x79d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.075338:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026345 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.075339:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.075344:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc02635a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x79d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.075355:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026345 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x79a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.075374:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc02635a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x79d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.075394:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc02635a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x79d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.075399:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.075401:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc02635a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x79d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.075437:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.075439:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.075443:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026345 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x79a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.075443:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.075447:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.075453:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.075458:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026345 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x79a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.075465:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.075466:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026345 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x79a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.075476:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634573b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+650:3628:x1870152993214336:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4324us (4651us total) trans 4294976218 rc 0/0 00010000:00010000:3.0:1783520936.075477:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.075483:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026345 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x79a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.075485:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026361 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x79e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.075487:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44713 00010000:00010000:2.0:1783520936.075514:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026345 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x79a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.075518:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.075524:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026361 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x79e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.075530:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517331f80/2, rc 0, transno 4294976218 00010000:00010000:3.0:1783520936.075537:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026361 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x79e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.075539:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.075545:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.075547:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517331f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993214336:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.075550:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.075557:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026361 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x79e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.075568:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026361 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x79e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.075597:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.075600:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+650:3628:x1870152993215360:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2769us (2866us total) trans 4294976217 rc 0/0 00000004:00080000:1.0:1783520936.075608:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503a6ad80/2, rc 0, transno 4294976217 00000100:00100000:2.0:1783520936.075611:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44716 00000100:00100000:0.0:1783520936.075622:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+650:3628:x1870152993215488:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2716us (2804us total) trans 4294976219 rc 0/0 00000100:00100000:1.0:1783520936.075624:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503a6ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993215360:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.075636:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44717 00000004:00080000:1.0:1783520936.075641:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e3b3100/2, rc 0, transno 4294976219 00000100:00100000:1.0:1783520936.075646:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e3b3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993215488:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.075662:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351dc41500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993217152:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.075681:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.075686:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.075690:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.075716:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993217408:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.075741:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e3b1500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993216128:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.075752:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.075758:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026368 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x79f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.075783:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993217152 00000100:00100000:3.0:1783520936.075788:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993217152 00000100:00100000:0.0:1783520936.075792:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1783520936.075801:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993217280:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.075820:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993217408 00000100:00100000:3.0:1783520936.075825:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993217408 00000100:00100000:1.0:1783520936.075829:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44723 00000100:00100000:0.0:1783520936.075835:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.075841:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e472300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+650:3626:x1870152993217152:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.075854:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1956 count 1 00000100:00100000:3.0:1783520936.075861:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993217280 00000100:00100000:3.0:1783520936.075866:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993217280 00010000:00010000:1.0:1783520936.075867:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02638b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7a4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.075881:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ebe73e3e/0xd5401cbabc021a9e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cbe expref: 357 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.075889:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a4:0x0].0x0 since=37s v=(0 0) 00000100:00100000:0.0:1783520936.075892:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993216256:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.075894:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a4:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.075897:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44724 00010000:00010000:1.0:1783520936.075903:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02638b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.075912:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44725 00000100:00100000:3.0:1783520936.075913:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+651:3628:x1870152993217408:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.075914:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02636f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.075928:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+651:3628:x1870152993217280:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.075929:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ebe73e3e/0xd5401cbabc021a9e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cbe expref: 357 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.075933:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1958 count 1 00010000:00010000:1.0:1783520936.075947:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ebe73e3e/0xd5401cbabc021a9e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cbe expref: 357 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.075949:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1957 count 1 00010000:00010000:3.0:1783520936.075951:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026392 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7a6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.075965:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026399 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7a5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.075982:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000552102e1/0xd5401cbabc0228b3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fa3 expref: 357 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.075995:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e471c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993217536:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.075997:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x7a6:0x0].0x0 since=36s v=(0 0) 00010000:00010000:2.0:1783520936.075997:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000000cbec3cc/0xd5401cbabc021aa5 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ccc expref: 357 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.076007:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.076015:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a5:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.076017:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026392 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.076021:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.076030:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026399 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.076042:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607eb8180 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.076051:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000552102e1/0xd5401cbabc0228b3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fa3 expref: 357 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.076058:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000000cbec3cc/0xd5401cbabc021aa5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ccc expref: 357 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.076064:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000552102e1/0xd5401cbabc0228b3 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fa3 expref: 357 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.076079:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000000cbec3cc/0xd5401cbabc021aa5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ccc expref: 357 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.076088:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee5f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993217664:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.076098:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503325c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993217792:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.076121:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435129670c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.076134:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872b40 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.076350:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.076422:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.076445:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002450432 00000100:00100000:2.0:1783520936.076450:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002450432 00000100:00100000:1.0:1783520936.076451:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002450560 00000100:00100000:1.0:1783520936.076456:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002450560 00010000:00010000:2.0:1783520936.076477:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005d902a4d/0xd5401cbabc021a82 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x79f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c78 expref: 359 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.076477:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000723b120a/0xd5401cbabc021a7b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c86 expref: 359 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.076524:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36309 00000100:00100000:1.0:1783520936.076533:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b6680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+359:2389:x1870153002450560:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520936.076538:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36310 00010000:00010000:1.0:1783520936.076545:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.076550:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b6300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+359:2387:x1870153002450432:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.076555:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000723b120a/0xd5401cbabc021a7b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7a0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c86 expref: 359 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.076559:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.076566:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005d902a4d/0xd5401cbabc021a82 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x79f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c78 expref: 359 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.076572:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000723b120a/0xd5401cbabc021a7b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7a0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c86 expref: 359 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.076578:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005d902a4d/0xd5401cbabc021a82 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x79f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c78 expref: 359 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.076600:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005d902a4d/0xd5401cbabc021a82 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x79f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c78 expref: 359 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.076602:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000723b120a/0xd5401cbabc021a7b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7a0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c86 expref: 359 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.076614:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x79f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.076618:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x79f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.076622:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026368 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.076622:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.076629:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.076634:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026368 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x79f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.076636:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02636f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.076651:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02636f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.076661:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.076674:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.076682:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026368 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.076690:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026368 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.076695:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b6300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+357:2387:x1870153002450432:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 146us (342us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.076699:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02636f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.076702:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b6680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+357:2389:x1870153002450560:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 168us (279us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.076707:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026368 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x79f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.076715:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36310 00010000:00010000:2.0:1783520936.076724:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02636f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.076725:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36309 00010000:00010000:0.0:1783520936.076726:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026368 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x79f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.076750:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02636f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.076759:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02636f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.076761:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.076765:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.076773:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026368 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x79f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.076782:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.076785:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026368 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x79f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.076788:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.076793:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.076798:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026368 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x79f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.076800:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02636f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.076818:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.076821:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026368 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x79f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.076824:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02636f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.076841:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026368 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x79f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.076844:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02636f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.076867:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02636f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.076868:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.076872:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.076876:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.076878:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02636f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.076884:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+652:3625:x1870152993216000:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2526us (3119us total) trans 4294976220 rc 0/0 00000100:00100000:2.0:1783520936.076895:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44719 00010000:00010000:0.0:1783520936.076899:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520936.076902:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515776680/2, rc 0, transno 4294976220 00010000:00010000:0.0:1783520936.076903:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.076908:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.076919:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515776680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993216000:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.076978:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363840d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+652:3626:x1870152993215872:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3033us (3269us total) trans 4294976221 rc 0/0 00000100:00100000:2.0:1783520936.076982:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e3b1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993217920:0@lo:6:osp-syn-1-0.0 00000004:00080000:1.0:1783520936.076988:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e3b1c00/2, rc 0, transno 4294976221 00000100:00100000:0.0:1783520936.076988:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44718 00000100:00100000:1.0:1783520936.076997:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e3b1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993215872:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.077017:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.077034:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e3b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993218048:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.077039:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993217920 00000100:00100000:0.0:1783520936.077043:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993217920 00000100:00100000:1.0:1783520936.077062:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.077070:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993218048 00000100:00100000:0.0:1783520936.077073:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993218048 00000100:00100000:2.0:1783520936.077083:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44726 00000100:00100000:2.0:1783520936.077093:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+653:3625:x1870152993217920:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.077098:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44727 00000100:00100000:3.0:1783520936.077110:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e3b0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+653:3626:x1870152993218048:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.077112:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1959 count 1 00002000:00080000:3.0:1783520936.077127:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1960 count 1 00010000:00010000:2.0:1783520936.077128:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0263a0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7a7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.077142:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0263a7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7a8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.077149:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000527fc16c/0xd5401cbabc021ab3 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ce8 expref: 355 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.077160:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000240854d7/0xd5401cbabc021ac8 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cf6 expref: 355 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.077162:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a7:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.077171:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.077175:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a8:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.077178:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.077182:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0263a7 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.077190:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0263a0 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.077207:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000240854d7/0xd5401cbabc021ac8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cf6 expref: 355 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.077218:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000527fc16c/0xd5401cbabc021ab3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ce8 expref: 355 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.077220:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000240854d7/0xd5401cbabc021ac8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cf6 expref: 355 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.077245:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee5500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993218176:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.077247:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000527fc16c/0xd5401cbabc021ab3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ce8 expref: 355 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.077262:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a83100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993218304:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.077287:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514219840 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.077293:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872e40 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.078042:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e471f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993216896:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.078052:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.078054:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351dc40700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993217024:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.078065:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02637d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.078069:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363840e680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993216768:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.078074:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026384 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.078084:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.078092:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026376 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.078108:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002450688 00000100:00100000:0.0:1783520936.078113:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.078115:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002450688 00010000:00010000:2.0:1783520936.078136:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000096f73924/0xd5401cbabc021a90 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ca2 expref: 356 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.078141:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002450816 00000100:00100000:1.0:1783520936.078147:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002450816 00010000:00010000:1.0:1783520936.078167:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000946c731f/0xd5401cbabc021a97 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cb0 expref: 357 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.078173:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002450944 00000100:00100000:2.0:1783520936.078178:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002450944 00010000:00010000:2.0:1783520936.078195:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000021ab238e/0xd5401cbabc021a89 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c94 expref: 358 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.078201:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36311 00000100:00100000:1.0:1783520936.078212:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36312 00000100:00100000:3.0:1783520936.078217:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b4700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+358:2389:x1870153002450688:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.078227:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032ea680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+358:2389:x1870153002450816:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.078228:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.078237:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36313 00010000:00010000:3.0:1783520936.078239:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000096f73924/0xd5401cbabc021a90 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7a2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ca2 expref: 358 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.078241:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.078251:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000946c731f/0xd5401cbabc021a97 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7a3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cb0 expref: 358 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.078253:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032e9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+358:2389:x1870153002450944:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.078261:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000096f73924/0xd5401cbabc021a90 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7a2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ca2 expref: 358 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.078275:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000946c731f/0xd5401cbabc021a97 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7a3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cb0 expref: 358 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.078281:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.078286:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000096f73924/0xd5401cbabc021a90 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7a2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ca2 expref: 358 pid: 8410 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.078292:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000021ab238e/0xd5401cbabc021a89 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7a1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c94 expref: 358 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.078297:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.078297:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000946c731f/0xd5401cbabc021a97 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7a3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cb0 expref: 357 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.078304:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.078312:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02637d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.078317:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000021ab238e/0xd5401cbabc021a89 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7a1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c94 expref: 357 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.078321:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02637d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.078321:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.078330:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.078342:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026384 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.078347:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000021ab238e/0xd5401cbabc021a89 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7a1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1c94 expref: 356 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.078355:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.078387:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026384 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.078389:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.078389:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02637d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.078398:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a1:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.078404:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b4700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+355:2389:x1870153002450688:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 191us (354us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.078410:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026376 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.078412:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02637d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.078415:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36311 00010000:00010000:2.0:1783520936.078435:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026376 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.078436:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02637d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.078451:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.078459:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02637d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.078478:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032ea680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+354:2389:x1870153002450816:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 254us (395us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.078482:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.078485:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.078489:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36312 00010000:00010000:2.0:1783520936.078508:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.078515:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02637d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.078524:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.078527:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02637d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.078534:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e471c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993217536:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.078535:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02637d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.078548:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02637d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.078554:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc02637d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.078555:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02638b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.078571:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.078573:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.078575:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.078579:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026384 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.078588:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026384 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.078598:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026384 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.078606:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026384 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.078626:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.079195:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.079199:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026384 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.079213:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.079217:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026384 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.079226:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026384 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.079233:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032e9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+353:2389:x1870153002450944:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 983us (1119us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.079234:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517332d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+655:3628:x1870152993216512:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4392us (4523us total) trans 4294976222 rc 0/0 00010000:00010000:3.0:1783520936.079241:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026384 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.079247:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36313 00010000:00010000:3.0:1783520936.079252:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026384 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.079252:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44721 00010000:00010000:3.0:1783520936.079271:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.079273:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.079275:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.079286:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026376 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.079295:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026376 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.079296:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517333100/2, rc 0, transno 4294976222 00010000:00010000:3.0:1783520936.079307:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026376 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.079314:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517333100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993216512:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.079327:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026376 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.079395:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635785050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+654:3625:x1870152993216640:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4398us (4490us total) trans 4294976223 rc 0/0 00000100:00100000:1.0:1783520936.079403:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44722 00010000:00010000:3.0:1783520936.079408:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.079421:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.079426:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026376 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.079432:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:3.0:1783520936.079434:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.079438:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026376 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.079444:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a80380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993218432:0@lo:6:osp-syn-1-0.0 00000004:00080000:1.0:1783520936.079444:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363d3a5500/2, rc 0, transno 4294976223 00010000:00010000:3.0:1783520936.079447:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026376 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.079453:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.079460:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993216640:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.079467:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026376 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.079476:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026376 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.079482:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.079526:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002451072 00000100:00100000:2.0:1783520936.079528:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e471c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993218560:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.079532:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002451072 00000100:00100000:0.0:1783520936.079549:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.079555:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000552102e1/0xd5401cbabc0228b3 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x7a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fa3 expref: 353 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.079566:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.079580:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.079593:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002451200 00000100:00100000:1.0:1783520936.079598:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002451200 00010000:00010000:1.0:1783520936.079615:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ebe73e3e/0xd5401cbabc021a9e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cbe expref: 354 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.079623:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993218432 00000100:00100000:2.0:1783520936.079629:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993218432 00000100:00100000:1.0:1783520936.079648:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002451328 00000100:00100000:1.0:1783520936.079653:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002451328 00000100:00100000:0.0:1783520936.079658:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.079662:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993218560 00000100:00100000:2.0:1783520936.079667:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993218560 00010000:00010000:1.0:1783520936.079671:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000527fc16c/0xd5401cbabc021ab3 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ce8 expref: 355 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.079684:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002451456 00000100:00100000:0.0:1783520936.079690:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002451456 00000100:00100000:2.0:1783520936.079697:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44728 00000100:00100000:1.0:1783520936.079702:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36314 00000100:00100000:2.0:1783520936.079709:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351dc40700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+655:3628:x1870152993218432:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.079711:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000000cbec3cc/0xd5401cbabc021aa5 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ccc expref: 356 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.079716:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032eb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+356:2390:x1870153002451072:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.079731:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00002000:00080000:2.0:1783520936.079732:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1961 count 1 00010000:00010000:1.0:1783520936.079739:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 1s ns: filter-lustre-OST0001_UUID lock: 00000000552102e1/0xd5401cbabc0228b3 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x7a6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fa3 expref: 356 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.079740:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36315 00010000:00010000:2.0:1783520936.079751:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc0263ae lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7a9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.079757:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638586450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+356:2388:x1870153002451200:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.079758:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000552102e1/0xd5401cbabc0228b3 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7a6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fa3 expref: 356 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.079772:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.079773:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.079775:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000097b2af14/0xd5401cbabc021eaa lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d04 expref: 356 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.079777:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.079781:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a6:0x0].0x0 since=1s v=(1 0) 00010000:00010000:3.0:1783520936.079782:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.079783:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 1s ns: filter-lustre-OST0001_UUID lock: 00000000ebe73e3e/0xd5401cbabc021a9e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7a4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cbe expref: 356 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.079792:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.079794:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a9:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.079803:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026392 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.079804:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.079810:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ebe73e3e/0xd5401cbabc021a9e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7a4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cbe expref: 356 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.079814:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc0263ae lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.079829:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026392 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7a6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.079844:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ebe73e3e/0xd5401cbabc021a9e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7a4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cbe expref: 356 pid: 8408 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.079850:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.079857:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000097b2af14/0xd5401cbabc021eaa lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d04 expref: 355 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.079858:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a4:0x0].0x0 since=1s v=(0 0) 00010000:00010000:0.0:1783520936.079866:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.079876:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02638b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.079881:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000097b2af14/0xd5401cbabc021eaa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d04 expref: 355 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.079883:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032eb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+355:2390:x1870153002451072:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 170us (432us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.079895:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02638b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.079901:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36314 00000100:00100000:3.0:1783520936.079911:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517333480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+656:3627:x1870152993216384:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5190us (5259us total) trans 4294976224 rc 0/0 00000100:00100000:1.0:1783520936.079924:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36316 00000100:00100000:2.0:1783520936.079930:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509175880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993218688:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.079930:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.079932:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44720 00000100:00100000:1.0:1783520936.079939:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032ebb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+354:2390:x1870153002451328:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.079949:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.079955:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 1s ns: filter-lustre-OST0001_UUID lock: 00000000527fc16c/0xd5401cbabc021ab3 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7a7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ce8 expref: 354 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.079957:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638586450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+354:2388:x1870153002451200:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 205us (409us total) trans 0 rc 0/0 00000004:00080000:2.0:1783520936.079962:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503a69880/2, rc 0, transno 4294976224 00000100:00100000:0.0:1783520936.079976:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36315 00000100:00100000:2.0:1783520936.079980:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503a69880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993216384:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.079981:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000527fc16c/0xd5401cbabc021ab3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7a7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ce8 expref: 354 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.080001:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a7:0x0].0x0 since=1s v=(0 0) 00010000:00010000:1.0:1783520936.080005:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.080010:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0263a0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.080021:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0263a0 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7a7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.080037:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36317 00000100:00100000:1.0:1783520936.080045:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1783520936.080049:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032e8700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+353:2389:x1870153002451456:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.080060:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509175f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993218816:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.080060:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.080068:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 1s ns: filter-lustre-OST0001_UUID lock: 000000000cbec3cc/0xd5401cbabc021aa5 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7a5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ccc expref: 353 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.080081:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000000cbec3cc/0xd5401cbabc021aa5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7a5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ccc expref: 353 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.080095:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a5:0x0].0x0 since=1s v=(0 0) 00000100:00100000:2.0:1783520936.080096:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872180 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.080101:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a5:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.080102:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.080108:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44729 00010000:00010000:0.0:1783520936.080111:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026399 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.080119:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026399 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7a5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.080121:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509177b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+655:3628:x1870152993218560:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.080131:0:35276:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000cbec3cc/0xd5401cbabc021aa5 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7a5:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ccc expref: 353 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.080139:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.080144:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00002000:00080000:3.0:1783520936.080146:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1962 count 1 00000100:00100000:2.0:1783520936.080149:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503325c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993217792:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.080163:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc0263b5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7aa:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.080165:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032ebb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+352:2390:x1870153002451328:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 231us (585us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.080170:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026399 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.080172:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032e8700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+352:2389:x1870153002451456:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 124us (512us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.080187:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36316 00000100:00100000:0.0:1783520936.080197:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36317 00010000:00010000:3.0:1783520936.080198:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000000c3a3bad/0xd5401cbabc021acf lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d12 expref: 352 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.080198:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026399 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.080219:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026399 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.080221:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7aa:0x0].0x0 since=38s v=(0 0) 00000100:00100000:0.0:1783520936.080223:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993218816 00000100:00100000:0.0:1783520936.080230:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993218816 00010000:00010000:1.0:1783520936.080231:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02638b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.080232:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7aa:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.080238:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026399 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.080240:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc0263b5 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.080249:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02638b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.080250:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44730 00000100:00100000:0.0:1783520936.080261:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e471f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+656:3626:x1870152993218816:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.080265:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026399 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.080277:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02638b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.080280:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1963 count 1 00010000:00010000:3.0:1783520936.080283:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000000c3a3bad/0xd5401cbabc021acf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d12 expref: 350 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.080293:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.080297:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02638b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.080300:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.080307:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000000c3a3bad/0xd5401cbabc021acf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d12 expref: 350 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.080309:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026399 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.080326:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.080330:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.080331:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.080334:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02638b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.080337:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026399 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.080338:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993218944:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.080348:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.080354:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02638b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.080383:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02638b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.080386:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026399 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.080394:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02638b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.080403:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026399 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.080405:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02638b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.080424:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d83c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.080424:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026399 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.080441:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.080445:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.080449:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.080451:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.080455:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.080458:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.080460:0:35275:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000552102e1/0xd5401cbabc0228b3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fa3 expref: 350 pid: 35275 timeout: 4230 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.080471:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0263bc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7ab:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.080474:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e472300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+657:3626:x1870152993217152:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4635us (4784us total) trans 4294976225 rc 0/0 00010000:00010000:2.0:1783520936.080477:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000527fc16c/0xd5401cbabc021ab3 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ce8 expref: 349 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.080481:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee5f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993217664:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.080509:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44723 00000100:00100000:2.0:1783520936.080513:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a83100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993218304:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.080515:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002472087e/0xd5401cbabc021ad6 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d20 expref: 348 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.080525:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026392 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.080529:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ab:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.080536:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0263a0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.080538:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ab:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.080545:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026392 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.080545:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351dc41500/2, rc 0, transno 4294976225 00010000:00010000:0.0:1783520936.080548:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0263bc lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.080560:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0263a0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.080561:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351dc41500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993217152:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.080570:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026392 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.080585:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0263a0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.080590:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002472087e/0xd5401cbabc021ad6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d20 expref: 348 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.080591:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026392 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.080598:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0263a0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.080610:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993219200:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.080619:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026392 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.080619:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002472087e/0xd5401cbabc021ad6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d20 expref: 348 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.080623:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0263a0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.080642:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.080654:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032ea680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993219072:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.080659:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520936.080665:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:1.0:1783520936.080678:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993219200 00000100:00100000:1.0:1783520936.080681:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993219200 00000100:00100000:0.0:1783520936.080684:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.080704:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44731 00000100:00100000:1.0:1783520936.080714:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+658:3626:x1870152993219200:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.080720:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00002000:00080000:1.0:1783520936.080726:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1964 count 1 00010000:00010000:3.0:1783520936.080727:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026392 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.080738:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.080742:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0263c3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7ac:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.080743:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026392 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.080760:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+658:3625:x1870152993217920:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3664us (3738us total) trans 4294976226 rc 0/0 00000100:00100000:0.0:1783520936.080764:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.080772:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002c6d1e32/0xd5401cbabc021add lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d2e expref: 348 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.080782:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44726 00010000:00010000:3.0:1783520936.080788:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026392 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.080790:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ac:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.080798:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ac:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.080806:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0263c3 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.080817:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026392 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.080821:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e3b1500/2, rc 0, transno 4294976226 00010000:00010000:1.0:1783520936.080829:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002c6d1e32/0xd5401cbabc021add lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d2e expref: 348 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.080831:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026392 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.080837:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e3b1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993217920:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.080854:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002c6d1e32/0xd5401cbabc021add lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d2e expref: 348 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.080860:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.080865:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0263a0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.080871:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993219328:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.080876:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.080880:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0263a0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.080885:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0263a0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.080889:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:3.0:1783520936.080895:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0263a0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.080901:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509176300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993219456:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.080902:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0263a0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.080904:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607eb8300 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.080929:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.080932:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.080934:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.080939:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.080946:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.080970:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993219456 00000100:00100000:0.0:1783520936.080975:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993219456 00000100:00100000:2.0:1783520936.080979:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002451584 00000100:00100000:2.0:1783520936.080986:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002451584 00010000:00010000:2.0:1783520936.081006:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000240854d7/0xd5401cbabc021ac8 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cf6 expref: 349 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.081009:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44732 00000100:00100000:0.0:1783520936.081023:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509177100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+658:3628:x1870152993219456:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.081038:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1965 count 1 00000100:00100000:2.0:1783520936.081043:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36318 00000100:00100000:2.0:1783520936.081054:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032e9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+349:2388:x1870153002451584:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.081054:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0263ca lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7ad:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.081058:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee5500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993218176:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.081070:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.081079:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 1s ns: filter-lustre-OST0001_UUID lock: 00000000240854d7/0xd5401cbabc021ac8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7a8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cf6 expref: 349 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.081079:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d144d97a/0xd5401cbabc021ae4 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d3c expref: 349 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.081080:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0263a7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7a8:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.081103:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ad:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.081108:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ad:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.081111:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000240854d7/0xd5401cbabc021ac8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7a8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cf6 expref: 349 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.081114:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0263ca lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.081146:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d144d97a/0xd5401cbabc021ae4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d3c expref: 349 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.081153:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000240854d7/0xd5401cbabc021ac8 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7a8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1cf6 expref: 349 pid: 12734 timeout: 4230 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.081163:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d144d97a/0xd5401cbabc021ae4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d3c expref: 349 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.081168:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a8:0x0].0x0 since=1s v=(0 0) 00010000:00010000:2.0:1783520936.081176:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a8:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.081180:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032eb100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993219584:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.081183:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0263a7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.081192:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0263a7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.081210:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.081221:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.081234:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032e9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+348:2388:x1870153002451584:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 181us (477us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.081246:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36318 00000100:00100000:3.0:1783520936.081258:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+659:3628:x1870152993217408:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5349us (5505us total) trans 4294976227 rc 0/0 00000100:00100000:3.0:1783520936.081267:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44724 00000004:00080000:2.0:1783520936.081280:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b5180/2, rc 0, transno 4294976227 00010000:00010000:3.0:1783520936.081291:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0263a7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.081292:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993217408:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.081306:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0263a7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.081315:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0263a7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.081323:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0263a7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.081345:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.081346:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509176680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993219712:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.081347:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.081355:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0263a7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.081391:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.081395:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0263a7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.081403:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0263a7 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.081416:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0263a7 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x7a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.081424:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0263a7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.081434:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.081439:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.081443:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.081446:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.081466:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993219712 00000100:00100000:1.0:1783520936.081471:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993219712 00000100:00100000:3.0:1783520936.081508:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e3b0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+661:3626:x1870152993218048:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4398us (4442us total) trans 4294976228 rc 0/0 00000100:00100000:1.0:1783520936.081516:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44733 00000004:00080000:0.0:1783520936.081522:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e3b2a00/2, rc 0, transno 4294976228 00000100:00100000:3.0:1783520936.081529:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44727 00000100:00100000:1.0:1783520936.081529:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032eb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+661:3628:x1870152993219712:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.081535:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+661:3628:x1870152993217280:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5611us (5700us total) trans 4294976229 rc 0/0 00000100:00100000:0.0:1783520936.081541:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e3b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993218048:0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.081544:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1966 count 1 00000100:00100000:2.0:1783520936.081548:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44725 00010000:00010000:1.0:1783520936.081559:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0263d1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7ae:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.081572:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d815045a/0xd5401cbabc022b0d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f31fd expref: 347 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.081574:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b7480/2, rc 0, transno 4294976229 00010000:00010000:1.0:1783520936.081585:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x7ae:0x0].0x0 since=37s v=(0 0) 00000100:00100000:0.0:1783520936.081585:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993217280:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.081593:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ae:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.081599:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0263d1 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.081602:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509174e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993219840:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.081611:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509174700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993219968:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.081631:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d815045a/0xd5401cbabc022b0d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31fd expref: 347 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.081643:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.081647:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d815045a/0xd5401cbabc022b0d lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31fd expref: 347 pid: 35276 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.081659:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993220096:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.081674:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.081702:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993219840 00000100:00100000:2.0:1783520936.081707:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993219840 00000100:00100000:0.0:1783520936.081722:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993219968 00000100:00100000:0.0:1783520936.081727:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993219968 00000100:00100000:1.0:1783520936.081731:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607eb8180 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.081732:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44734 00000100:00100000:2.0:1783520936.081742:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+661:3625:x1870152993219840:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.081753:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1967 count 1 00000100:00100000:1.0:1783520936.081760:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44735 00010000:00010000:2.0:1783520936.081768:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0263d8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7af:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.081770:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032e8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+661:3626:x1870152993219968:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.081780:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000eaf2b1e9/0xd5401cbabc021af9 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d58 expref: 347 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.081788:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7af:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.081792:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7af:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.081796:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0263d8 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.081802:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1968 count 1 00010000:00010000:1.0:1783520936.081815:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0263df lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7b0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.081816:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000eaf2b1e9/0xd5401cbabc021af9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d58 expref: 347 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.081826:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000eaf2b1e9/0xd5401cbabc021af9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d58 expref: 347 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.081834:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000221214a3/0xd5401cbabc021b00 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d66 expref: 347 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.081837:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509175500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993220224:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.081841:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b0:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.081844:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.081849:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0263df lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.081866:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872e40 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.081867:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000221214a3/0xd5401cbabc021b00 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d66 expref: 347 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.081876:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000221214a3/0xd5401cbabc021b00 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d66 expref: 347 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.081887:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a7b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993220352:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.081908:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff94363865b240 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.082271:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.082299:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509175880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993218688:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.082302:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002451712 00000100:00100000:3.0:1783520936.082307:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002451712 00010000:00010000:2.0:1783520936.082318:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc0263ae lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.082324:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000097b2af14/0xd5401cbabc021eaa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7a9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d04 expref: 348 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.082353:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36319 00000100:00100000:3.0:1783520936.082381:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a4e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+348:2390:x1870153002451712:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.082388:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.082393:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000097b2af14/0xd5401cbabc021eaa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7a9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d04 expref: 348 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.082402:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000097b2af14/0xd5401cbabc021eaa lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7a9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d04 expref: 348 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.082415:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000097b2af14/0xd5401cbabc021eaa lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7a9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d04 expref: 348 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.082430:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.082435:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7a9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.082439:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc0263ae lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.082454:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc0263ae lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.082521:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.082550:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc0263ae lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.082552:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a4e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+347:2390:x1870153002451712:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 191us (280us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.082569:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36319 00010000:00010000:0.0:1783520936.082569:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc0263ae lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.082582:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc0263ae lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.082591:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc0263ae lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.082612:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.082620:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.082631:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc0263ae lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.082640:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.082643:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc0263ae lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.082652:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc0263ae lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.082667:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc0263ae lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.082675:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc0263ae lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.082693:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.082696:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.082699:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.082731:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351dc40700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+662:3628:x1870152993218432:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3025us (3248us total) trans 4294976230 rc 0/0 00000100:00100000:0.0:1783520936.082739:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44728 00000004:00080000:2.0:1783520936.082803:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943509a80380/2, rc 0, transno 4294976230 00000100:00100000:2.0:1783520936.082818:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a80380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993218432:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.082869:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1783520936.082878:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032e8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993220480:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.082905:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.082929:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993220480 00000100:00100000:0.0:1783520936.082932:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993220480 00000100:00100000:0.0:1783520936.082953:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44736 00000100:00100000:0.0:1783520936.082960:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509175880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+662:3628:x1870152993220480:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.082975:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1969 count 1 00010000:00010000:0.0:1783520936.082987:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0263e6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7b1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.082998:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000bc907fee/0xd5401cbabc021b07 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d74 expref: 346 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.083005:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b1:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.083009:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.083013:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0263e6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.083036:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000bc907fee/0xd5401cbabc021b07 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d74 expref: 346 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.083047:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000bc907fee/0xd5401cbabc021b07 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d74 expref: 346 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.083058:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032e9500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993220608:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.083096:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514219780 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.084533:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032ea680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993219072:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.084554:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0263bc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.084584:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993218944:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.084595:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.084613:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc0263b5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.084624:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993219328:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.084628:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.084648:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0263c3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.084656:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002451968 00000100:00100000:1.0:1783520936.084661:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002451968 00010000:00010000:1.0:1783520936.084686:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002c6d1e32/0xd5401cbabc021add lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d2e expref: 347 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.084704:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032eb100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993219584:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.084718:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002451840 00010000:00010000:0.0:1783520936.084718:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0263ca lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.084724:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002451840 00010000:00010000:1.0:1783520936.084741:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002472087e/0xd5401cbabc021ad6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d20 expref: 348 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.084749:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36320 00000100:00100000:0.0:1783520936.084760:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e006d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+348:2389:x1870153002451968:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.084770:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.084776:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002c6d1e32/0xd5401cbabc021add lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ac:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d2e expref: 348 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.084779:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36321 00000100:00100000:1.0:1783520936.084793:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+348:2388:x1870153002451840:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.084802:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002c6d1e32/0xd5401cbabc021add lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7ac:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d2e expref: 348 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.084808:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.084816:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002472087e/0xd5401cbabc021ad6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ab:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d20 expref: 348 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.084826:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002c6d1e32/0xd5401cbabc021add lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7ac:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d2e expref: 348 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.084831:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002472087e/0xd5401cbabc021ad6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7ab:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d20 expref: 348 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.084850:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ac:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.084856:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002472087e/0xd5401cbabc021ad6 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7ab:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d20 expref: 347 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.084856:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ac:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.084864:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0263c3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.084871:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ab:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.084876:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ab:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.084877:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0263c3 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.084882:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0263bc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.084895:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0263bc lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.084907:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.084915:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0263c3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.084918:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.084925:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0263c3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.084932:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0263c3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.084938:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0263c3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.084946:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e006d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+346:2389:x1870153002451968:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 186us (352us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.084947:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+346:2388:x1870153002451840:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 157us (321us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.084952:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.084964:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520936.084965:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36320 00010000:00010000:3.0:1783520936.084968:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0263c3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.084974:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.084977:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0263c3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.084978:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36321 00010000:00010000:3.0:1783520936.084988:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0263c3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.084998:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0263c3 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.085003:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0263c3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.085018:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.085020:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.085021:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.085043:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.085086:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002452224 00000100:00100000:2.0:1783520936.085091:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002452224 00010000:00010000:2.0:1783520936.085112:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d144d97a/0xd5401cbabc021ae4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d3c expref: 345 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.085116:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.085116:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0263bc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.085127:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0263bc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.085136:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0263bc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.085140:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002452096 00000100:00100000:2.0:1783520936.085143:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002452096 00010000:00010000:0.0:1783520936.085145:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0263bc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.085157:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000000c3a3bad/0xd5401cbabc021acf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d12 expref: 346 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.085166:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520936.085180:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+663:3626:x1870152993219200:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4466us (4539us total) trans 4294976231 rc 0/0 00000100:00100000:2.0:1783520936.085180:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36322 00000100:00100000:2.0:1783520936.085189:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638587c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+346:2390:x1870153002452224:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000004:00080000:0.0:1783520936.085192:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363d3a4000/2, rc 0, transno 4294976231 00000100:00100000:3.0:1783520936.085193:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44731 00010000:00010000:2.0:1783520936.085202:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.085203:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993219200:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.085208:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d144d97a/0xd5401cbabc021ae4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ad:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d3c expref: 346 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.085219:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d144d97a/0xd5401cbabc021ae4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7ad:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d3c expref: 346 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.085230:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.085237:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0263bc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.085240:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d144d97a/0xd5401cbabc021ae4 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7ad:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d3c expref: 346 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.085258:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.085260:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ad:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.085265:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0263bc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.085265:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ad:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.085275:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.085277:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0263ca lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.085285:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0263bc lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.085289:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0263ca lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.085294:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1783520936.085309:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032eb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993220736:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.085313:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.085340:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638587c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+345:2390:x1870153002452224:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 152us (296us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.085344:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e471f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+663:3626:x1870152993218816:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5085us (5246us total) trans 4294976232 rc 0/0 00010000:00010000:3.0:1783520936.085347:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0263bc lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.085351:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36322 00000100:00100000:0.0:1783520936.085379:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44730 00010000:00010000:3.0:1783520936.085384:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0263bc lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.085395:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:0.0:1783520936.085411:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943509175f80/2, rc 0, transno 4294976232 00010000:00010000:3.0:1783520936.085416:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.085419:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.085423:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.085424:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509175f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993218816:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.085426:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002452352 00000100:00100000:2.0:1783520936.085429:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002452352 00000100:00100000:3.0:1783520936.085437:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993220736 00000100:00100000:3.0:1783520936.085441:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993220736 00010000:00010000:2.0:1783520936.085445:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000221214a3/0xd5401cbabc021b00 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d66 expref: 345 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.085473:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36323 00000100:00100000:3.0:1783520936.085477:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44737 00000100:00100000:0.0:1783520936.085477:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1783520936.085483:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36324 00000100:00100000:1.0:1783520936.085488:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511adad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+345:2388:x1870153002452096:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.085501:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993220864:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.085502:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+663:3627:x1870152993220736:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.085509:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511adbb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+345:2389:x1870153002452352:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.085521:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.085521:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00002000:00080000:3.0:1783520936.085526:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1970 count 1 00010000:00010000:1.0:1783520936.085532:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000000c3a3bad/0xd5401cbabc021acf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7aa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d12 expref: 345 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.085533:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000221214a3/0xd5401cbabc021b00 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7b0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d66 expref: 345 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.085533:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.085548:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0263ed lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7b2:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.085561:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000000c3a3bad/0xd5401cbabc021acf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7aa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d12 expref: 345 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.085562:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000221214a3/0xd5401cbabc021b00 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7b0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d66 expref: 345 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.085567:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c7effd80/0xd5401cbabc021b0e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d82 expref: 345 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.085571:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993220864 00000100:00100000:0.0:1783520936.085577:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993220864 00010000:00010000:3.0:1783520936.085587:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b2:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.085593:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000c3a3bad/0xd5401cbabc021acf lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7aa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d12 expref: 345 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.085594:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b2:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.085604:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44738 00010000:00010000:2.0:1783520936.085612:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.085614:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0263ed lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.085616:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7aa:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.085616:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032e8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+664:3626:x1870152993220864:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.085617:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.085627:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7aa:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.085629:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0263df lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.085633:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1971 count 1 00010000:00010000:1.0:1783520936.085639:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc0263b5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.085643:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0263df lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7b0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.085648:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0263f4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7b3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.085653:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc0263b5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.085659:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c7effd80/0xd5401cbabc021b0e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d82 expref: 344 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.085663:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.085667:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ecf1bb5d/0xd5401cbabc021b1c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d90 expref: 344 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.085683:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511adbb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+344:2389:x1870153002452352:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 191us (415us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.085689:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c7effd80/0xd5401cbabc021b0e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d82 expref: 344 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.085691:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.085692:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b3:0x0].0x0 since=38s v=(0 0) 00000100:00100000:2.0:1783520936.085695:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36324 00010000:00010000:0.0:1783520936.085699:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.085708:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0263f4 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.085714:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993220992:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.085721:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511adad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+343:2388:x1870153002452096:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 236us (607us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.085732:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36323 00010000:00010000:0.0:1783520936.085738:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ecf1bb5d/0xd5401cbabc021b1c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d90 expref: 343 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.085749:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ecf1bb5d/0xd5401cbabc021b1c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d90 expref: 342 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.085760:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032e9c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993221120:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.085762:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.085777:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435129670c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.085780:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993220096:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.085790:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514219840 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.085796:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc0263b5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.085802:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0263d1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.085807:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002452608 00010000:00010000:3.0:1783520936.085808:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc0263b5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.085812:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002452608 00010000:00010000:3.0:1783520936.085826:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc0263b5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.085832:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d815045a/0xd5401cbabc022b0d lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31fd expref: 343 pid: 35276 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.085834:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc0263b5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.085849:0:35274:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000221214a3/0xd5401cbabc021b00 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d66 expref: 343 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.085850:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.085864:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.085864:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a7b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993220352:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.085876:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520936.085876:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002452480 00000100:00100000:0.0:1783520936.085880:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002452480 00010000:00010000:2.0:1783520936.085882:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc0263b5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.085883:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0263df lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.085895:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000eaf2b1e9/0xd5401cbabc021af9 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d58 expref: 343 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.085900:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.085910:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc0263b5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.085910:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0263df lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.085921:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36325 00000100:00100000:0.0:1783520936.085933:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509174a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+343:2389:x1870153002452608:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.085934:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0263df lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.085937:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509177b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+665:3628:x1870152993218560:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5819us (6372us total) trans 4294976233 rc 0/0 00010000:00010000:2.0:1783520936.085945:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc0263b5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.085947:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.085952:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44729 00010000:00010000:0.0:1783520936.085956:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d815045a/0xd5401cbabc022b0d lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7ae:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31fd expref: 343 pid: 35276 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.085963:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0263df lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.085968:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc0263b5 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.085969:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d815045a/0xd5401cbabc022b0d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7ae:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31fd expref: 343 pid: 35276 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.085979:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0263df lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.085982:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc0263b5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.085988:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d815045a/0xd5401cbabc022b0d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7ae:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31fd expref: 343 pid: 35276 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.086004:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ae:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.086007:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ae:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.086008:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.086012:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to regular blp list, 0 blwis in pool 00010000:00010000:0.0:1783520936.086012:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0263d1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.086013:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0263df lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.086021:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.086023:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.086024:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0263d1 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.086028:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0263df lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.086035:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0263df lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.086042:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.086045:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0263df lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.086053:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0263df lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.086061:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509174a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+342:2389:x1870153002452608:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 131us (300us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.086069:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36325 00010000:00010000:2.0:1783520936.086079:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.086081:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.086083:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36326 00010000:00010000:2.0:1783520936.086084:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.086089:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509176a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+341:2387:x1870153002452480:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000004:00080000:2.0:1783520936.086107:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e471c00/2, rc 0, transno 4294976233 00000004:00080000:0.0:1783520936.086112:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943509174700/2, rc 0, transno 4294976234 00000100:00100000:1.0:1783520936.086113:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032e8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+665:3626:x1870152993219968:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4343us (4437us total) trans 4294976234 rc 0/0 00000100:00100000:2.0:1783520936.086122:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e471c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993218560:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.086122:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509174700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993219968:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.086124:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44735 00010000:00010000:0.0:1783520936.086143:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.086150:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000eaf2b1e9/0xd5401cbabc021af9 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7af:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d58 expref: 341 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.086155:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0263d1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.086171:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0263d1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.086173:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000eaf2b1e9/0xd5401cbabc021af9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7af:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d58 expref: 341 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.086194:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0263d1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.086196:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7af:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.086204:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7af:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.086208:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.086209:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0263d1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.086210:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0263d8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7af:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.086229:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:0.0:1783520936.086229:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0263d8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7af:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.086235:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002452736 00000100:00100000:3.0:1783520936.086241:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002452736 00000100:00100000:2.0:1783520936.086241:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993221376:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.086246:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993221248:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.086256:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.086261:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.086267:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0263d1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.086275:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.086283:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbaa00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.086289:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0263d1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.086291:0:35275:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000eaf2b1e9/0xd5401cbabc021af9 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7af:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d58 expref: 342 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.086304:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0263d1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.086312:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509175500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993220224:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.086316:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0263d1 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.086323:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0263d1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.086323:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993221248 00010000:00010000:2.0:1783520936.086329:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0263d8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.086329:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993221248 00010000:00010000:3.0:1783520936.086339:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.086340:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032eb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+666:3628:x1870152993219712:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4809us (4899us total) trans 4294976235 rc 0/0 00010000:00010000:3.0:1783520936.086343:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.086347:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.086352:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.086356:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44733 00000100:00100000:0.0:1783520936.086390:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44739 00000100:00100000:0.0:1783520936.086397:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032eb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+665:3626:x1870152993221248:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.086400:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000bc907fee/0xd5401cbabc021b07 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d74 expref: 341 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.086413:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993221376 00002000:00080000:0.0:1783520936.086413:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1972 count 1 00000100:00100000:1.0:1783520936.086419:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993221376 00000100:00100000:3.0:1783520936.086426:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36327 00000004:00080000:2.0:1783520936.086427:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943509176680/2, rc 0, transno 4294976235 00010000:00010000:0.0:1783520936.086428:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0263fb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7b4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.086436:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e006d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+341:2389:x1870153002452736:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.086441:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509176680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993219712:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.086441:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003811e1a9/0xd5401cbabc021b23 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d9e expref: 341 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.086446:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.086447:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44740 00010000:00010000:3.0:1783520936.086455:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000bc907fee/0xd5401cbabc021b07 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7b1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d74 expref: 341 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.086458:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b4:0x0].0x0 since=38s v=(0 0) 00000100:00100000:1.0:1783520936.086463:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509175500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+666:3628:x1870152993221376:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.086468:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.086473:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000bc907fee/0xd5401cbabc021b07 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7b1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d74 expref: 341 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.086476:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0263fb lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.086480:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1973 count 1 00010000:00010000:3.0:1783520936.086486:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b1:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.086486:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:3.0:1783520936.086508:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.086508:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026402 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7b5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.086511:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993221504:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.086515:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0263e6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.086521:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003811e1a9/0xd5401cbabc021b23 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d9e expref: 341 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.086540:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0263e6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7b1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.086540:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.086541:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000acc718bd/0xd5401cbabc021b2a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1dac expref: 341 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.086546:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003811e1a9/0xd5401cbabc021b23 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d9e expref: 341 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.086566:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.086567:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b5:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.086573:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b5:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.086573:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032ea300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993221632:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.086585:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026402 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.086593:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e006d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+341:2389:x1870153002452736:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 157us (384us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.086607:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.086612:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36327 00010000:00010000:0.0:1783520936.086626:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.086633:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000acc718bd/0xd5401cbabc021b2a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1dac expref: 340 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.086645:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993221504 00010000:00010000:1.0:1783520936.086646:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000acc718bd/0xd5401cbabc021b2a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1dac expref: 340 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.086648:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509176a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+340:2387:x1870153002452480:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 558us (803us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.086652:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993221504 00000100:00100000:0.0:1783520936.086666:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36326 00000100:00100000:1.0:1783520936.086670:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ad9880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993221760:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.086680:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0263d8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.086681:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44741 00000100:00100000:3.0:1783520936.086695:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+667:3627:x1870152993221504:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.086704:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0263d8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.086708:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000bc907fee/0xd5401cbabc021b07 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d74 expref: 339 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.086709:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1974 count 1 00000100:00100000:1.0:1783520936.086709:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435190663c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.086718:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0263d8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.086724:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032e9500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993220608:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.086726:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026409 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7b6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.086726:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0263d8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.086737:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0263e6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.086748:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c4e98cac/0xd5401cbabc0229fc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f30ec expref: 338 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.086758:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.086763:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0263e6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.086774:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x7b6:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.086782:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.086783:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0263e6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.086785:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026409 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.086804:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0263e6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.086806:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.086812:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0263d8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.086819:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0263e6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.086820:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.086824:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0263d8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.086837:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+668:3625:x1870152993219840:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5097us (5195us total) trans 4294976236 rc 0/0 00010000:00010000:3.0:1783520936.086840:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0263d8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.086845:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.086853:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44734 00010000:00010000:3.0:1783520936.086859:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0263d8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.086864:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0263d8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.086865:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943509174e00/2, rc 0, transno 4294976236 00000100:00100000:1.0:1783520936.086881:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509174e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993219840:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.086884:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.086888:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0263e6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.086898:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.086901:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0263e6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.086905:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0263e6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.086912:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0263e6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.086917:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0263e6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.086919:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993221888:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.086925:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509175880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+668:3628:x1870152993220480:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3962us (4016us total) trans 4294976237 rc 0/0 00010000:00010000:3.0:1783520936.086933:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.086934:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44736 00010000:00010000:3.0:1783520936.086936:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:2.0:1783520936.086937:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435032e8a80/2, rc 0, transno 4294976237 00010000:00010000:3.0:1783520936.086940:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.086947:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032e8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993220480:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.086949:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.086957:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.086962:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0263ca lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.086971:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0263ca lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.086980:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0263ca lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.086983:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1783520936.086992:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993222016:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.086999:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0263ca lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.087014:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.087020:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.087023:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.087028:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0263ca lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.087037:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.087042:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0263ca lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.087051:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993221888 00010000:00010000:2.0:1783520936.087051:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0263ca lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.087057:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993221888 00010000:00010000:2.0:1783520936.087068:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0263ca lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.087076:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0263ca lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.087088:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509177100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+669:3628:x1870152993219456:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6069us (6150us total) trans 4294976238 rc 0/0 00000100:00100000:3.0:1783520936.087094:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993222016 00010000:00010000:2.0:1783520936.087095:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.087095:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44732 00010000:00010000:2.0:1783520936.087099:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.087100:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993222016 00010000:00010000:2.0:1783520936.087103:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.087105:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44742 00000100:00100000:1.0:1783520936.087116:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+669:3625:x1870152993221888:12345-0@lo:6:osp-syn-1-0.0 00000004:00080000:2.0:1783520936.087125:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943509176300/2, rc 0, transno 4294976238 00002000:00080000:1.0:1783520936.087129:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1975 count 1 00000100:00100000:2.0:1783520936.087135:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509176300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993219456:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.087143:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026410 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7b7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.087156:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000037f985b8/0xd5401cbabc021b38 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1dc8 expref: 338 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.087165:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b7:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.087169:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.087174:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026410 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.087178:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44743 00000100:00100000:2.0:1783520936.087179:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509a83100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993222144:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.087192:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+669:3625:x1870152993222016:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.087207:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000037f985b8/0xd5401cbabc021b38 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1dc8 expref: 338 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.087217:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.087222:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000037f985b8/0xd5401cbabc021b38 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1dc8 expref: 338 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.087226:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1976 count 1 00000100:00100000:1.0:1783520936.087239:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511adaa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993222272:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.087244:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026417 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7b8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.087249:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993222144 00000100:00100000:0.0:1783520936.087255:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993222144 00010000:00010000:3.0:1783520936.087268:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a41af913/0xd5401cbabc021b46 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1dd6 expref: 338 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.087274:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b8:0x0].0x0 since=38s v=(0 0) 00000100:00100000:1.0:1783520936.087275:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff94363865b240 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.087278:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.087282:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026417 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.087283:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44744 00000100:00100000:0.0:1783520936.087294:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503372a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+670:3627:x1870152993222144:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.087308:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1977 count 1 00010000:00010000:3.0:1783520936.087309:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a41af913/0xd5401cbabc021b46 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1dd6 expref: 338 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.087324:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a41af913/0xd5401cbabc021b46 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1dd6 expref: 338 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.087324:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02641e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7b9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.087353:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993222400:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.087354:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000079bfebc2/0xd5401cbabc021b4d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1de4 expref: 338 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.087390:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b9:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.087394:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.087398:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02641e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.087418:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000079bfebc2/0xd5401cbabc021b4d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1de4 expref: 338 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.087437:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000079bfebc2/0xd5401cbabc021b4d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1de4 expref: 338 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.087447:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032e9500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993222528:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.087469:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d83c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.087471:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.087516:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c4e98cac/0xd5401cbabc0229fc lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30ec expref: 338 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.087528:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c4e98cac/0xd5401cbabc0229fc lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30ec expref: 338 pid: 8410 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.087539:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993222656:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.087577:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57d80 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.089440:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993220992:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.089464:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0263ed lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.089947:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032e9c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993221120:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.089964:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0263f4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.089993:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032ea300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993221632:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.090009:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0263fb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.091239:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.091309:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.091332:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.091449:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.091475:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.091483:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ad9880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993221760:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.091522:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026402 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.091554:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002452864 00000100:00100000:0.0:1783520936.091559:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511adaa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993222272:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.091562:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002452864 00000100:00100000:2.0:1783520936.091574:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.091583:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026410 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.091604:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c7effd80/0xd5401cbabc021b0e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d82 expref: 339 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.091618:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.091622:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002453120 00000100:00100000:0.0:1783520936.091628:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002453120 00000100:00100000:3.0:1783520936.091632:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002452992 00000100:00100000:3.0:1783520936.091638:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002452992 00010000:00010000:0.0:1783520936.091655:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003811e1a9/0xd5401cbabc021b23 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d9e expref: 341 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.091657:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ecf1bb5d/0xd5401cbabc021b1c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d90 expref: 341 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.091682:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002453248 00000100:00100000:3.0:1783520936.091686:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002453248 00000100:00100000:2.0:1783520936.091692:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.091705:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002453376 00010000:00010000:3.0:1783520936.091708:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000acc718bd/0xd5401cbabc021b2a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1dac expref: 342 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.091709:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002453376 00000100:00100000:2.0:1783520936.091727:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002453504 00010000:00010000:0.0:1783520936.091728:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000037f985b8/0xd5401cbabc021b38 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1dc8 expref: 343 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.091734:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002453504 00000100:00100000:3.0:1783520936.091742:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002453632 00000100:00100000:3.0:1783520936.091747:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002453632 00000100:00100000:0.0:1783520936.091749:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002453760 00000100:00100000:0.0:1783520936.091753:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002453760 00010000:00010000:2.0:1783520936.091764:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a41af913/0xd5401cbabc021b46 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1dd6 expref: 346 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.091767:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000079bfebc2/0xd5401cbabc021b4d lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1de4 expref: 346 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.091771:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c4e98cac/0xd5401cbabc0229fc lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x7b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30ec expref: 346 pid: 8410 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.091814:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36328 00000100:00100000:1.0:1783520936.091839:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+346:2387:x1870153002452864:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.091863:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.091872:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c7effd80/0xd5401cbabc021b0e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7b2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d82 expref: 346 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.091893:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c7effd80/0xd5401cbabc021b0e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7b2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d82 expref: 346 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.091906:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36329 00000100:00100000:0.0:1783520936.091920:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+346:2388:x1870153002452992:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.091929:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c7effd80/0xd5401cbabc021b0e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7b2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d82 expref: 346 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.091933:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.091943:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ecf1bb5d/0xd5401cbabc021b1c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7b3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d90 expref: 346 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.091954:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ecf1bb5d/0xd5401cbabc021b1c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7b3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d90 expref: 345 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.091956:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.091975:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.091984:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0263ed lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.091989:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ecf1bb5d/0xd5401cbabc021b1c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7b3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d90 expref: 345 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.092004:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.092006:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0263ed lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.092009:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.092026:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0263f4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.092031:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36330 00000100:00100000:2.0:1783520936.092049:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36331 00000100:00100000:3.0:1783520936.092052:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638584850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+344:2387:x1870153002453120:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.092057:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0263f4 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.092065:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.092068:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503372300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+344:2387:x1870153002453248:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.092074:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003811e1a9/0xd5401cbabc021b23 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7b4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d9e expref: 344 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.092081:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.092086:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.092087:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.092092:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003811e1a9/0xd5401cbabc021b23 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7b4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d9e expref: 344 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.092094:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000acc718bd/0xd5401cbabc021b2a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7b5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1dac expref: 344 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.092120:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000acc718bd/0xd5401cbabc021b2a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7b5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1dac expref: 344 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.092124:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003811e1a9/0xd5401cbabc021b23 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7b4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1d9e expref: 344 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.092148:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000acc718bd/0xd5401cbabc021b2a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7b5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1dac expref: 343 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.092149:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b4:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.092154:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+343:2388:x1870153002452992:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 237us (819us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.092158:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b4:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.092169:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+343:2387:x1870153002452864:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 331us (926us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.092173:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0263fb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.092179:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b5:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.092189:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36329 00010000:00010000:2.0:1783520936.092200:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.092205:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0263fb lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.092211:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36328 00010000:00010000:2.0:1783520936.092213:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026402 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.092231:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36332 00010000:00010000:2.0:1783520936.092242:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026402 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.092249:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+340:2390:x1870153002453376:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.092263:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.092267:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.092272:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36333 00010000:00010000:0.0:1783520936.092275:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000037f985b8/0xd5401cbabc021b38 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7b7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1dc8 expref: 340 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.092277:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.092289:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000037f985b8/0xd5401cbabc021b38 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7b7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1dc8 expref: 340 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.092303:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638584850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+340:2387:x1870153002453120:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 256us (992us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.092309:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000037f985b8/0xd5401cbabc021b38 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7b7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1dc8 expref: 340 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.092315:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+340:2387:x1870153002453632:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.092318:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503372300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+340:2387:x1870153002453248:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 250us (862us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.092333:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36330 00010000:00010000:1.0:1783520936.092340:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.092344:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b7:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.092348:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36331 00010000:00010000:0.0:1783520936.092351:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.092353:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000079bfebc2/0xd5401cbabc021b4d lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7b9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1de4 expref: 338 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.092408:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026410 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.092426:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026410 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.092434:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000079bfebc2/0xd5401cbabc021b4d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7b9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1de4 expref: 337 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.092435:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36334 00010000:00010000:0.0:1783520936.092444:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.092461:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503370e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+337:2389:x1870153002453760:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520936.092462:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36335 00000100:00100000:0.0:1783520936.092469:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+337:2390:x1870153002453376:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 223us (990us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.092484:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+337:2387:x1870153002453504:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.092488:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.092519:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36332 00010000:00010000:3.0:1783520936.092522:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.092522:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c4e98cac/0xd5401cbabc0229fc lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x7b6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30ec expref: 337 pid: 8410 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.092528:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.092534:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a41af913/0xd5401cbabc021b46 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7b8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1dd6 expref: 336 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.092541:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0263f4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.092545:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.092560:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c4e98cac/0xd5401cbabc0229fc lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7b6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30ec expref: 336 pid: 8410 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.092575:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02641e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.092581:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0263f4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.092593:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a41af913/0xd5401cbabc021b46 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7b8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1dd6 expref: 336 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.092613:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.092627:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02641e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7b9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.092629:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.092642:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0263f4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.092647:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.092651:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026409 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.092669:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7b8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.092681:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026417 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.092685:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0263f4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.092694:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026409 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7b6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.092706:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026417 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7b8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.092720:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.092720:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.092723:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.092732:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.092741:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0263f4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.092755:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+336:2387:x1870153002453632:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 452us (1151us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.092772:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbebe00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520936.092773:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503370e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+336:2389:x1870153002453760:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 313us (1072us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.092776:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36333 00010000:00010000:3.0:1783520936.092786:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0263f4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.092799:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36334 00010000:00010000:3.0:1783520936.092803:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0263f4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.092816:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0263ed lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.092822:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0263f4 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.092824:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0263ed lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.092857:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0263ed lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.092861:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0263f4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.092868:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0263ed lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.092870:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.092887:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032e8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+671:3626:x1870152993220864:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7266us (7345us total) trans 4294976239 rc 0/0 00000100:00100000:2.0:1783520936.092905:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+334:2387:x1870153002453504:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 420us (1324us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.092919:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44738 00010000:00010000:1.0:1783520936.092923:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.092924:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36335 00010000:00010000:3.0:1783520936.092948:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.092953:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0263ed lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.092959:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4600 already unused, calling callback (ffffffffc0bf9e90) 00000004:00080000:0.0:1783520936.092961:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435032ea680/2, rc 0, transno 4294976239 00010000:00010000:3.0:1783520936.092963:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0263ed lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.092971:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0263ed lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.092977:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993220864:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.092980:0:35276:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c4e98cac/0xd5401cbabc0229fc lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30ec expref: 333 pid: 8410 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.092987:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0263ed lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.092998:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0263ed lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.093001:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993222656:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.093033:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026409 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.093035:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.093038:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.093041:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.093049:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026409 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.093059:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026409 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.093060:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.093071:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026409 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.093074:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0263fb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.093082:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435032eb100/2, rc 0, transno 4294976240 00000100:00100000:1.0:1783520936.093088:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+671:3627:x1870152993220736:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7596us (7734us total) trans 4294976240 rc 0/0 00010000:00010000:2.0:1783520936.093092:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026409 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.093110:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032eb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993220736:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.093119:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0263fb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.093127:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44737 00000100:00100000:0.0:1783520936.093135:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:3.0:1783520936.093152:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0263fb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.093157:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032e9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993222784:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.093161:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0263fb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.093167:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.093176:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.093181:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026409 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.093186:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.093189:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026409 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.093195:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026409 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.093204:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026409 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.093208:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026409 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.093228:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.093234:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.093238:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.093241:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.093261:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.093275:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.093285:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0263fb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.093294:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.093300:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0263fb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.093316:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0263fb lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.093326:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993222912:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.093335:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0263fb lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.093346:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0263fb lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.093391:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032eb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+672:3626:x1870152993221248:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6961us (7086us total) trans 4294976242 rc 0/0 00000100:00100000:3.0:1783520936.093398:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44739 00000100:00100000:0.0:1783520936.093417:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.093424:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.093430:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.093435:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.093466:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a41af913/0xd5401cbabc021b46 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1dd6 expref: 332 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.093468:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943508ee5f80/2, rc 0, transno 4294976242 00000100:00100000:0.0:1783520936.093478:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993221248:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.093488:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993222400:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.093509:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993222784 00000100:00100000:3.0:1783520936.093515:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993222784 00010000:00010000:2.0:1783520936.093535:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026417 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.093545:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032ebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993223040:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.093548:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026417 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.093557:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993222912 00010000:00010000:1.0:1783520936.093557:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026410 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.093559:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026417 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.093566:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993222912 00000100:00100000:0.0:1783520936.093591:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.093593:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44745 00010000:00010000:1.0:1783520936.093611:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026410 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.093620:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026417 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.093623:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026410 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.093653:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993223040 00010000:00010000:1.0:1783520936.093656:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026410 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.093657:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032eaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+673:3627:x1870152993222784:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.093661:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026417 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.093662:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993223040 00002000:00080000:3.0:1783520936.093691:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1978 count 1 00000100:00100000:0.0:1783520936.093701:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44746 00010000:00010000:3.0:1783520936.093716:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026425 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7ba:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.093717:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032ea300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+674:3625:x1870152993222912:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.093726:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520936.093734:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00002000:00080000:0.0:1783520936.093736:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1979 count 1 00010000:00010000:3.0:1783520936.093750:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000cbd664fa/0xd5401cbabc021b54 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1df2 expref: 331 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.093755:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ba:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.093755:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02642c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7bb:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.093762:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ba:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.093769:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026425 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.093781:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520936.093782:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000072c69c30/0xd5401cbabc021b5b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e00 expref: 331 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.093785:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026410 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.093806:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.093806:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7bb:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.093814:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7bb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.093815:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026410 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.093822:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02642c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.093828:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026410 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.093847:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026410 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.093858:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026410 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.093876:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363d3a7100/2, rc 0, transno 4294976243 00000100:00100000:2.0:1783520936.093889:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+676:3625:x1870152993222016:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6695us (6866us total) trans 4294976243 rc 0/0 00010000:00010000:3.0:1783520936.093891:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520936.093892:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993222016:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.093898:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+676:3625:x1870152993221888:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6774us (6932us total) trans 4294976244 rc 0/0 00010000:00010000:3.0:1783520936.093902:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026417 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.093908:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363d3a4e00/2, rc 0, transno 4294976244 00000100:00100000:2.0:1783520936.093919:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44743 00000100:00100000:0.0:1783520936.093922:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993221888:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.093935:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44742 00010000:00010000:3.0:1783520936.093940:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.093949:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026417 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.093949:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44747 00010000:00010000:3.0:1783520936.093962:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026417 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.093966:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032e8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+674:3626:x1870152993223040:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.093987:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1980 count 1 00010000:00010000:3.0:1783520936.093990:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026417 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.093994:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993223168:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.093999:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026433 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7bc:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.094008:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026417 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.094022:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009b6e151e/0xd5401cbabc021b62 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e0e expref: 331 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.094022:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000079bfebc2/0xd5401cbabc021b4d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1de4 expref: 331 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.094030:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.094035:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.094041:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.094046:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.094058:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.094063:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:2.0:1783520936.094070:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7bc:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.094074:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7bc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.094078:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026433 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.094080:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993223296:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.094090:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000cbd664fa/0xd5401cbabc021b54 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1df2 expref: 330 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.094090:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032e9500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993222528:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.094107:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009b6e151e/0xd5401cbabc021b62 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e0e expref: 330 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.094119:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.094128:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02641e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.094132:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009b6e151e/0xd5401cbabc021b62 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e0e expref: 330 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.094133:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000cbd664fa/0xd5401cbabc021b54 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1df2 expref: 330 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.094151:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993223168 00000100:00100000:0.0:1783520936.094158:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993223168 00000100:00100000:0.0:1783520936.094181:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993223296 00000100:00100000:2.0:1783520936.094189:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503373480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993223552:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.094189:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993223296 00000100:00100000:3.0:1783520936.094195:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993223424:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.094195:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02641e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.094214:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44748 00010000:00010000:1.0:1783520936.094226:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02641e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.094233:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+676:3627:x1870152993223168:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.094242:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872480 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.094248:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435129670c0 going to sleep for 11 seconds 00002000:00080000:0.0:1783520936.094259:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1981 count 1 00010000:00010000:1.0:1783520936.094260:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02641e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.094271:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026402 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.094275:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02641e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.094280:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02643a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7bd:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.094292:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+676:3627:x1870152993221504:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7598us (7748us total) trans 4294976241 rc 0/0 00010000:00010000:3.0:1783520936.094303:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026402 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.094327:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44741 00010000:00010000:0.0:1783520936.094335:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000565eb3f4/0xd5401cbabc021b69 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e1c expref: 330 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.094338:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026402 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.094341:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.094351:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44749 00010000:00010000:0.0:1783520936.094384:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7bd:0x0].0x0 since=38s v=(0 0) 00000100:00100000:2.0:1783520936.094389:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+675:3627:x1870152993223296:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.094391:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7bd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.094399:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02643a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.094401:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1982 count 1 00010000:00010000:3.0:1783520936.094403:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026402 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.094419:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026441 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7be:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.094432:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.094437:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a9c92c5c/0xd5401cbabc0228eb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fdb expref: 330 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.094439:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02641e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.094448:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363d3a7b80/2, rc 0, transno 4294976241 00010000:00010000:3.0:1783520936.094461:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.094461:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x7be:0x0].0x0 since=37s v=(0 0) 00000100:00100000:0.0:1783520936.094465:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993221504:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.094470:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02641e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.094471:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7be:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.094481:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026441 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.094485:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02641e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.094522:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02641e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.094522:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a9c92c5c/0xd5401cbabc0228eb lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fdb expref: 330 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.094529:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1783520936.094541:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517209880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993223680:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.094545:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02641e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.094569:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a9c92c5c/0xd5401cbabc0228eb lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fdb expref: 330 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.094585:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503370380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993223808:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.094595:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.094599:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.094608:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.094620:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.094650:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.094663:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.094669:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026402 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.094677:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.094684:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026402 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.094695:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026402 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.094697:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000072c69c30/0xd5401cbabc021b5b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e00 expref: 330 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.094722:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000072c69c30/0xd5401cbabc021b5b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e00 expref: 330 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.094727:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026402 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.094739:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026402 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.094742:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032e9880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993223936:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.094786:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.094793:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.094802:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.094819:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509175500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+677:3628:x1870152993221376:12345-0@lo:6:osp-syn-1-0.0 Request processed in 8357us (8466us total) trans 4294976246 rc 0/0 00000100:00100000:3.0:1783520936.094833:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44740 00000004:00080000:0.0:1783520936.094838:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943508ee6a00/2, rc 0, transno 4294976246 00000100:00100000:2.0:1783520936.094855:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872b40 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.094862:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993221376:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.094882:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993223680 00000100:00100000:3.0:1783520936.094889:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993223680 00000100:00100000:2.0:1783520936.094934:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44750 00000100:00100000:2.0:1783520936.094953:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517208000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+677:3626:x1870152993223680:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.094959:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00002000:00080000:2.0:1783520936.094972:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1983 count 1 00000100:00100000:0.0:1783520936.094978:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351720a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993224064:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.094990:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026448 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7bf:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.095005:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009a12e525/0xd5401cbabc021b7e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e38 expref: 330 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.095010:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000565eb3f4/0xd5401cbabc021b69 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e1c expref: 330 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.095015:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7bf:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.095024:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7bf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.095032:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026448 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.095035:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000565eb3f4/0xd5401cbabc021b69 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e1c expref: 330 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.095053:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.095068:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993224192:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.095078:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009a12e525/0xd5401cbabc021b7e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e38 expref: 330 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.095098:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009a12e525/0xd5401cbabc021b7e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e38 expref: 330 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.095116:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503371500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993224320:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.095121:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6600 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.095162:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993224064 00000100:00100000:0.0:1783520936.095163:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.095168:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993224064 00000100:00100000:2.0:1783520936.095168:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351c872e40 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.095231:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44751 00000100:00100000:3.0:1783520936.095249:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635787c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+678:3625:x1870152993224064:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520936.095267:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1984 count 1 00010000:00010000:3.0:1783520936.095286:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02644f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7c0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.095304:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006f29b4ae/0xd5401cbabc021b8c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e46 expref: 330 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.095311:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503372a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+678:3627:x1870152993222144:12345-0@lo:6:osp-syn-1-0.0 Request processed in 8011us (8089us total) trans 4294976245 rc 0/0 00010000:00010000:3.0:1783520936.095318:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c0:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.095324:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c0:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.095326:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44744 00010000:00010000:3.0:1783520936.095330:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02644f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.095435:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943509a83100/2, rc 0, transno 4294976245 00010000:00010000:2.0:1783520936.095446:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006f29b4ae/0xd5401cbabc021b8c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e46 expref: 330 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.095455:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509a83100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993222144:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.095476:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006f29b4ae/0xd5401cbabc021b8c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e46 expref: 330 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.095488:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351664b100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993224448:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.095584:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d83c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.095589:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1783520936.095606:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351720a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993224576:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.095664:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.095711:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993224576 00000100:00100000:3.0:1783520936.095717:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993224576 00000100:00100000:0.0:1783520936.095766:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44752 00000100:00100000:0.0:1783520936.095785:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517208e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+678:3628:x1870152993224576:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.095801:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1985 count 1 00010000:00010000:0.0:1783520936.095820:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026456 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7c1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.095839:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e02ec6da/0xd5401cbabc021b93 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e54 expref: 330 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.095848:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c1:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.095851:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.095856:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026456 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.095887:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e02ec6da/0xd5401cbabc021b93 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e54 expref: 330 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.095904:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e02ec6da/0xd5401cbabc021b93 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e54 expref: 330 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.095921:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517209500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993224704:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.095973:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.097017:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503373480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993223552:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.097055:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026433 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.097223:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032e9880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993223936:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.097254:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02642c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.098223:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993223424:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.098240:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503370380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993223808:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.098276:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026441 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.098277:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026425 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.098291:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993224192:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.098322:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.098342:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02643a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.098406:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002454144 00000100:00100000:2.0:1783520936.098412:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.098415:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002454144 00000100:00100000:3.0:1783520936.098443:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503371500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993224320:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.098462:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000cbd664fa/0xd5401cbabc021b54 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1df2 expref: 331 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.098472:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.098475:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026448 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.098486:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002454528 00000100:00100000:1.0:1783520936.098505:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002454528 00000100:00100000:2.0:1783520936.098519:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.098520:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002453888 00000100:00100000:0.0:1783520936.098526:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002453888 00010000:00010000:1.0:1783520936.098542:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006f29b4ae/0xd5401cbabc021b8c lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e46 expref: 333 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.098553:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002454016 00000100:00100000:2.0:1783520936.098563:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002454016 00000100:00100000:3.0:1783520936.098568:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351664b100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993224448:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.098571:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009b6e151e/0xd5401cbabc021b62 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e0e expref: 333 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.098587:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36336 00010000:00010000:3.0:1783520936.098596:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02644f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.098597:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000072c69c30/0xd5401cbabc021b5b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e00 expref: 334 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.098609:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638587850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+334:2390:x1870153002454144:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.098626:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36337 00000100:00100000:0.0:1783520936.098645:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516648e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+334:2390:x1870153002454528:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.098649:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.098661:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000cbd664fa/0xd5401cbabc021b54 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ba:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1df2 expref: 334 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.098664:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.098668:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36338 00010000:00010000:0.0:1783520936.098680:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006f29b4ae/0xd5401cbabc021b8c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7c0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e46 expref: 334 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.098689:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516648000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+334:2387:x1870153002453888:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.098709:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000cbd664fa/0xd5401cbabc021b54 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7ba:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1df2 expref: 334 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.098710:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.098711:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.098720:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006f29b4ae/0xd5401cbabc021b8c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7c0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e46 expref: 334 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.098728:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009b6e151e/0xd5401cbabc021b62 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7bc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e0e expref: 334 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.098745:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.098770:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cbd664fa/0xd5401cbabc021b54 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7ba:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1df2 expref: 334 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.098775:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.098788:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f29b4ae/0xd5401cbabc021b8c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7c0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e46 expref: 333 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.098790:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009b6e151e/0xd5401cbabc021b62 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7bc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e0e expref: 333 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.098795:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ba:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.098815:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ba:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.098835:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026425 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.098842:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.098848:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009b6e151e/0xd5401cbabc021b62 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7bc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e0e expref: 332 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.098849:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.098854:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026425 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.098865:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02644f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.098882:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.098905:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02644f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.098908:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7bc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.098917:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7bc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.098923:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026433 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.098929:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.098934:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026433 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.098937:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.098964:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.098985:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002454272 00000100:00100000:3.0:1783520936.098993:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002454272 00000100:00100000:1.0:1783520936.099005:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638587850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+332:2390:x1870153002454144:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 395us (692us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.099006:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516648000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+332:2387:x1870153002453888:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 317us (536us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.099008:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516648e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+332:2390:x1870153002454528:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 360us (592us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.099022:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a9c92c5c/0xd5401cbabc0228eb lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fdb expref: 332 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.099041:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36336 00000100:00100000:2.0:1783520936.099056:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36338 00000100:00100000:0.0:1783520936.099065:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36337 00000100:00100000:3.0:1783520936.099092:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002454400 00000100:00100000:3.0:1783520936.099097:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002454400 00000100:00100000:2.0:1783520936.099109:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002454656 00000100:00100000:0.0:1783520936.099114:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002454784 00000100:00100000:2.0:1783520936.099118:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002454656 00010000:00010000:3.0:1783520936.099119:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000565eb3f4/0xd5401cbabc021b69 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e1c expref: 330 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.099121:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002454784 00000100:00100000:1.0:1783520936.099143:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36339 00000100:00100000:3.0:1783520936.099150:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36340 00010000:00010000:2.0:1783520936.099157:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009a12e525/0xd5401cbabc021b7e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e38 expref: 332 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.099161:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351664b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+332:2388:x1870153002454016:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.099162:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e02ec6da/0xd5401cbabc021b93 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e54 expref: 332 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.099166:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee4000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+332:2387:x1870153002454272:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.099186:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.099191:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.099199:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000072c69c30/0xd5401cbabc021b5b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7bb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e00 expref: 332 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.099206:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a9c92c5c/0xd5401cbabc0228eb lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7be:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fdb expref: 332 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.099245:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36341 00010000:00010000:1.0:1783520936.099251:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000072c69c30/0xd5401cbabc021b5b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7bb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e00 expref: 332 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.099266:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36342 00000100:00100000:2.0:1783520936.099272:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee4a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+332:2388:x1870153002454400:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.099291:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a9c92c5c/0xd5401cbabc0228eb lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7be:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fdb expref: 332 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.099300:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee4e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+332:2387:x1870153002454656:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.099306:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.099310:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000072c69c30/0xd5401cbabc021b5b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7bb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e00 expref: 332 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.099321:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000565eb3f4/0xd5401cbabc021b69 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7bd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e1c expref: 332 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.099331:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a9c92c5c/0xd5401cbabc0228eb lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7be:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fdb expref: 332 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.099334:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.099353:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009a12e525/0xd5401cbabc021b7e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7bf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e38 expref: 332 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.099419:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7bb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.099436:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7bb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.099438:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7be:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.099457:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7be:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.099458:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02642c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.099469:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009a12e525/0xd5401cbabc021b7e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7bf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e38 expref: 330 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.099473:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026441 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.099484:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000565eb3f4/0xd5401cbabc021b69 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7bd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e1c expref: 330 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.099529:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02642c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.099550:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026441 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.099566:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009a12e525/0xd5401cbabc021b7e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7bf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e38 expref: 330 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.099567:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000565eb3f4/0xd5401cbabc021b69 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7bd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e1c expref: 330 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.099592:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.099598:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.099598:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7bd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.099605:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7bf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.099608:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7bd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.099611:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7bf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.099616:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02643a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.099618:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026448 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.099634:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351664b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+328:2388:x1870153002454016:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 476us (1111us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.099642:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee4000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+328:2387:x1870153002454272:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 472us (927us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.099669:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026448 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.099670:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02643a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.099672:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36339 00000100:00100000:3.0:1783520936.099700:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36340 00000100:00100000:1.0:1783520936.099720:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36343 00000100:00100000:1.0:1783520936.099730:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638585c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+326:2388:x1870153002454784:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.099738:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.099741:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.099742:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.099751:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e02ec6da/0xd5401cbabc021b93 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7c1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e54 expref: 326 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.099778:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e02ec6da/0xd5401cbabc021b93 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7c1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e54 expref: 326 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.099783:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee4e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+326:2387:x1870153002454656:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 493us (1011us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.099792:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee4a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+326:2388:x1870153002454400:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 523us (1046us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.099798:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517209500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993224704:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.099829:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36342 00000100:00100000:2.0:1783520936.099838:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36341 00010000:00010000:3.0:1783520936.099846:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026456 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7c1:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.099848:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e02ec6da/0xd5401cbabc021b93 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7c1:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e54 expref: 325 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.099866:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02642c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.099873:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026433 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.099883:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.099905:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.099907:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02642c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.099908:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026433 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.099915:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026456 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.099955:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02642c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.099961:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026456 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.099965:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026433 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.099993:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02642c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.100017:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026433 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.100020:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.100027:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026456 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.100034:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.100047:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:1.0:1783520936.100057:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638585c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+323:2388:x1870153002454784:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 322us (1184us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.100072:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026456 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.100077:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36343 00010000:00010000:3.0:1783520936.100087:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026456 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.100093:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026456 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.100110:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520936.100117:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02642c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.100125:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.100131:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02642c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.100140:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02642c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.100156:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02642c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.100164:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc02642c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.100215:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032ea300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+680:3625:x1870152993222912:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6496us (6794us total) trans 4294976247 rc 0/0 00010000:00010000:3.0:1783520936.100220:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520936.100225:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44746 00010000:00010000:3.0:1783520936.100228:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026433 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.100231:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032e8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+680:3626:x1870152993223040:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6268us (6649us total) trans 4294976248 rc 0/0 00010000:00010000:3.0:1783520936.100259:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185c00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520936.100263:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44747 00010000:00010000:3.0:1783520936.100267:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026433 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.100281:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026433 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.100281:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435032e8000/2, rc 0, transno 4294976247 00000100:00100000:0.0:1783520936.100307:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993222912:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.100317:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026433 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.100325:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026433 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.100344:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435032ebb80/2, rc 0, transno 4294976248 00010000:00010000:3.0:1783520936.100349:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.100353:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.100415:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032ebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993223040:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.100453:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.100468:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.100514:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:2.0:1783520936.100519:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.100532:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351720b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993224960:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.100563:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351720bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993224832:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.100567:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026425 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.100582:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026425 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.100594:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.100605:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026425 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.100616:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026425 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.100625:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517208e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+679:3628:x1870152993224576:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4844us (4957us total) trans 4294976249 rc 0/0 00000100:00100000:2.0:1783520936.100640:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44752 00010000:00010000:1.0:1783520936.100653:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:0.0:1783520936.100664:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993224960 00010000:00010000:2.0:1783520936.100670:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00000100:00100000:0.0:1783520936.100671:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993224960 00010000:00010000:2.0:1783520936.100680:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026456 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.100692:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.100704:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.100710:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026456 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.100715:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993224832 00000100:00100000:0.0:1783520936.100721:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993224832 00010000:00010000:2.0:1783520936.100727:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026456 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.100748:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026456 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.100758:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026456 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.100761:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351720a300/2, rc 0, transno 4294976249 00000100:00100000:0.0:1783520936.100772:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44753 00000100:00100000:0.0:1783520936.100783:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351720b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+681:3626:x1870152993224960:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.100795:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351720a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993224576:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.100799:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.100806:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026425 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.100815:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5200 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520936.100821:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032eaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+681:3627:x1870152993222784:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7206us (7592us total) trans 4294976250 rc 0/0 00010000:00010000:2.0:1783520936.100823:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026425 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.100829:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.100851:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026425 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.100851:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44745 00000100:00100000:0.0:1783520936.100855:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000004:00080000:3.0:1783520936.100864:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435032e9c00/2, rc 0, transno 4294976250 00000100:00100000:0.0:1783520936.100876:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517209c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993225088:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.100887:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032e9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993222784:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.100888:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026425 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.100899:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44754 00010000:00010000:2.0:1783520936.100916:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026425 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.100926:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026441 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.100931:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+680:3628:x1870152993224832:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.100941:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.100951:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.100958:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.100962:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026441 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.100964:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00002000:00080000:1.0:1783520936.100971:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1986 count 1 00010000:00010000:3.0:1783520936.100978:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026441 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.100982:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993225088 00000100:00100000:2.0:1783520936.100992:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993225088 00010000:00010000:3.0:1783520936.100993:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026441 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.100997:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02645d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7c2:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.101031:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003c08875b/0xd5401cbabc021b9a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e62 expref: 322 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.101032:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44755 00010000:00010000:3.0:1783520936.101036:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.101046:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c2:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.101051:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:2.0:1783520936.101051:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517209500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+681:3625:x1870152993225088:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.101054:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.101061:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026441 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.101066:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02645d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.101070:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993225216:0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.101089:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1988 count 1 00010000:00010000:3.0:1783520936.101109:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.101113:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026441 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.101121:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026441 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.101126:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026464 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7c4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.101133:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.101146:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003c08875b/0xd5401cbabc021b9a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e62 expref: 322 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.101179:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993225216 00000100:00100000:0.0:1783520936.101190:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993225216 00010000:00010000:2.0:1783520936.101206:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000144fa680/0xd5401cbabc021ba8 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e7e expref: 322 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.101219:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026441 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.101219:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c4:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.101222:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003c08875b/0xd5401cbabc021b9a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e62 expref: 322 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.101238:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c4:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.101238:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44756 00010000:00010000:2.0:1783520936.101265:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026464 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.101267:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513bc0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+682:3627:x1870152993225216:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.101273:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026441 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.101297:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ad9c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993225344:192.168.204.33@tcp:104: 00002000:00080000:0.0:1783520936.101301:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1989 count 1 00010000:00010000:0.0:1783520936.101323:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02646b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7c5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.101341:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006dce8fff/0xd5401cbabc021bb6 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e8c expref: 322 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.101347:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.101347:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c5:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.101352:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.101354:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.101384:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02646b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.101400:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000144fa680/0xd5401cbabc021ba8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e7e expref: 322 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.101405:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.101413:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006dce8fff/0xd5401cbabc021bb6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e8c expref: 322 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.101427:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000144fa680/0xd5401cbabc021ba8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e7e expref: 322 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.101434:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026448 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.101437:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006dce8fff/0xd5401cbabc021bb6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e8c expref: 322 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.101447:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02643a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.101453:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516649500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993225472:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.101487:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026448 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.101513:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02643a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.101521:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc2300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993225600:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.101522:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026448 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.101527:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02643a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.101546:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d83c0 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.101562:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026448 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.101562:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02643a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.101571:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.101598:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520936.101604:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520936.101609:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520936.101617:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026448 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.101627:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1987 count 1 00010000:00010000:3.0:1783520936.101628:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.101634:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026448 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.101642:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026448 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.101647:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026472 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7c3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.101662:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026448 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.101669:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000bbb00b33/0xd5401cbabc021ba1 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e70 expref: 322 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.101687:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026448 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.101699:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c3:0x0].0x0 since=38s v=(0 0) 00000100:00100000:1.0:1783520936.101708:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+683:3627:x1870152993223168:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7477us (7679us total) trans 4294976251 rc 0/0 00010000:00010000:0.0:1783520936.101710:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.101717:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.101732:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02643a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.101734:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026472 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.101746:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44748 00010000:00010000:0.0:1783520936.101784:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000bbb00b33/0xd5401cbabc021ba1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e70 expref: 322 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.101802:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000bbb00b33/0xd5401cbabc021ba1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e70 expref: 322 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.101803:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb400 already unused, calling callback (ffffffffc0bf9e90) 00000004:00080000:1.0:1783520936.101807:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b5500/2, rc 0, transno 4294976251 00010000:00010000:3.0:1783520936.101811:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02643a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.101831:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993223168:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.101850:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02643a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.101861:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993225728:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.101876:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02643a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.101879:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+683:3627:x1870152993223296:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7487us (7758us total) trans 4294976252 rc 0/0 00000004:00080000:1.0:1783520936.101881:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b6300/2, rc 0, transno 4294976252 00000100:00100000:1.0:1783520936.101900:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993223296:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.101905:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.101912:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44749 00010000:00010000:3.0:1783520936.101925:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc02643a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.101947:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124700c0 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.101963:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.101964:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02644f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.101969:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.101974:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.101983:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02644f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.101985:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.102003:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02644f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.102011:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02644f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.102018:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511adbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993225856:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.102022:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:2.0:1783520936.102037:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.102039:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511adad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993225984:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.102054:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.102062:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02644f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.102066:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.102075:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.102081:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02644f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.102091:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02644f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.102107:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.102113:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02644f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.102125:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc02644f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.102142:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993225856 00000100:00100000:0.0:1783520936.102150:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993225856 00010000:00010000:2.0:1783520936.102165:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.102175:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.102188:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.102222:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993225984 00000100:00100000:0.0:1783520936.102229:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993225984 00000100:00100000:1.0:1783520936.102234:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44757 00000100:00100000:1.0:1783520936.102255:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513bc1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+686:3628:x1870152993225856:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.102275:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517208000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+686:3626:x1870152993223680:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7329us (7685us total) trans 4294976253 rc 0/0 00000100:00100000:0.0:1783520936.102284:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44758 00000100:00100000:3.0:1783520936.102296:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44750 00000004:00080000:1.0:1783520936.102302:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517209880/2, rc 0, transno 4294976253 00000100:00100000:0.0:1783520936.102303:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511adaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+686:3626:x1870152993225984:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.102326:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517209880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993223680:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.102327:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635787c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+685:3625:x1870152993224064:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7080us (7274us total) trans 4294976254 rc 0/0 00002000:00080000:0.0:1783520936.102327:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1991 count 1 00010000:00010000:0.0:1783520936.102352:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026479 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7c7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.102353:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44751 00010000:00010000:0.0:1783520936.102423:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a794a83f/0xd5401cbabc021bc4 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ea8 expref: 322 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.102434:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c7:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.102438:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.102442:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026479 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.102467:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a794a83f/0xd5401cbabc021bc4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ea8 expref: 322 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.102474:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a794a83f/0xd5401cbabc021bc4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ea8 expref: 322 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.102484:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc1500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993226112:192.168.204.33@tcp:104: 00000004:00080000:1.0:1783520936.102563:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351720a680/2, rc 0, transno 4294976254 00000100:00100000:1.0:1783520936.102582:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351720a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993224064:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.102595:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1783520936.102605:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514219840 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.102611:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ad9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993226240:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.102659:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00080000:0.0:1783520936.102689:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1990 count 1 00000100:00100000:2.0:1783520936.102697:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993226240 00000100:00100000:2.0:1783520936.102705:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993226240 00010000:00010000:0.0:1783520936.102710:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026480 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7c6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.102724:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d42d5e00/0xd5401cbabc022b45 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3235 expref: 322 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.102730:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x7c6:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.102732:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.102734:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026480 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.102746:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44759 00000100:00100000:2.0:1783520936.102763:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+685:3625:x1870152993226240:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.102764:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d42d5e00/0xd5401cbabc022b45 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3235 expref: 322 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.102769:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00002000:00080000:2.0:1783520936.102783:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1992 count 1 00010000:00010000:0.0:1783520936.102788:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d42d5e00/0xd5401cbabc022b45 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3235 expref: 322 pid: 8409 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.102793:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ad8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993226368:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.102805:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026487 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7c8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.102825:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ae76f4cd/0xd5401cbabc021bcb lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1eb6 expref: 322 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.102825:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc3100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993226496:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.102839:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c8:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.102849:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c8:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.102849:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.102859:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026487 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.102890:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514219780 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.102895:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ae76f4cd/0xd5401cbabc021bcb lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1eb6 expref: 322 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.102899:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993226368 00010000:00010000:2.0:1783520936.102912:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ae76f4cd/0xd5401cbabc021bcb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1eb6 expref: 322 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.102929:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351664ad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993226624:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.102960:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993226368 00000100:00100000:2.0:1783520936.102983:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57d80 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.103269:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44760 00000100:00100000:0.0:1783520936.103283:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+686:3628:x1870152993226368:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.103299:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ad9c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993225344:192.168.204.33@tcp:104: 00002000:00080000:0.0:1783520936.103300:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1993 count 1 00000100:00100000:1.0:1783520936.103322:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.103327:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02648e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7c9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.103346:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02645d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.103347:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f590c184/0xd5401cbabc021bd2 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ec4 expref: 322 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.103355:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516649500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993225472:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.103445:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c9:0x0].0x0 since=38s v=(0 0) 00000100:00100000:1.0:1783520936.103455:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.103459:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026464 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.103459:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.103467:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02648e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.103486:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002454912 00000100:00100000:3.0:1783520936.103505:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002454912 00000100:00100000:1.0:1783520936.103509:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002455040 00010000:00010000:0.0:1783520936.103516:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f590c184/0xd5401cbabc021bd2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ec4 expref: 323 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.103518:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002455040 00010000:00010000:0.0:1783520936.103539:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f590c184/0xd5401cbabc021bd2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ec4 expref: 324 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.103553:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000144fa680/0xd5401cbabc021ba8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e7e expref: 324 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.103560:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc3800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993226752:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.103567:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc2300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993225600:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.103569:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003c08875b/0xd5401cbabc021b9a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e62 expref: 324 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.103611:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02646b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.103638:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36344 00000100:00100000:1.0:1783520936.103654:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+324:2387:x1870153002455040:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520936.103666:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36345 00010000:00010000:1.0:1783520936.103671:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.103679:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c016cc0 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.103680:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000144fa680/0xd5401cbabc021ba8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7c4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e7e expref: 324 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.103706:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000144fa680/0xd5401cbabc021ba8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7c4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e7e expref: 324 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.103729:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000144fa680/0xd5401cbabc021ba8 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7c4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e7e expref: 324 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.103745:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.103748:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.103753:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026464 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.103761:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026464 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.103788:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+323:2388:x1870153002454912:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.103789:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.103798:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.103805:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003c08875b/0xd5401cbabc021b9a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7c2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e62 expref: 323 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.103813:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026464 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.103821:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003c08875b/0xd5401cbabc021b9a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7c2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e62 expref: 323 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.103828:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+323:2387:x1870153002455040:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 176us (380us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.103863:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.103873:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026464 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.103876:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36344 00010000:00010000:3.0:1783520936.103893:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003c08875b/0xd5401cbabc021b9a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7c2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e62 expref: 323 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.103898:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026464 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.103909:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002455168 00000100:00100000:1.0:1783520936.103917:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002455168 00010000:00010000:3.0:1783520936.103927:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.103930:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026464 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.103934:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.103944:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006dce8fff/0xd5401cbabc021bb6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e8c expref: 322 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.103952:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02645d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.103980:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02645d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.103990:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.103998:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.104005:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026464 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.104013:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36346 00010000:00010000:3.0:1783520936.104016:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.104024:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026464 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.104031:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513bc0a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+322:2387:x1870153002455168:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.104036:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026464 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.104044:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.104058:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006dce8fff/0xd5401cbabc021bb6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7c5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e8c expref: 322 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.104070:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.104095:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006dce8fff/0xd5401cbabc021bb6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7c5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e8c expref: 322 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.104120:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993225728:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.104122:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006dce8fff/0xd5401cbabc021bb6 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7c5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e8c expref: 322 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.104144:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026464 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.104152:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026464 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.104153:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.104157:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026472 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.104164:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517209500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+687:3625:x1870152993225088:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3112us (3235us total) trans 4294976255 rc 0/0 00010000:00010000:1.0:1783520936.104176:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.104200:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02646b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.104214:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.104214:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44755 00010000:00010000:1.0:1783520936.104215:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02646b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.104223:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.104249:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002455296 00000100:00100000:0.0:1783520936.104257:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002455296 00010000:00010000:1.0:1783520936.104259:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.104284:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000bbb00b33/0xd5401cbabc021ba1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e70 expref: 322 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.104295:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517209c00/2, rc 0, transno 4294976255 00010000:00010000:3.0:1783520936.104326:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.104335:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517209c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993225088:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.104341:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36347 00000100:00100000:1.0:1783520936.104351:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513bc0a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+322:2387:x1870153002455168:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 322us (499us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.104395:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.104401:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36346 00000100:00100000:0.0:1783520936.104412:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513bc1880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+322:2388:x1870153002455296:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.104418:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02646b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.104429:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+321:2388:x1870153002454912:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 646us (1113us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.104451:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02646b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.104458:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36345 00010000:00010000:0.0:1783520936.104461:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.104477:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000bbb00b33/0xd5401cbabc021ba1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7c3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e70 expref: 321 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.104479:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02646b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.104504:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02645d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.104508:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:2.0:1783520936.104517:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02646b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.104520:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000bbb00b33/0xd5401cbabc021ba1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7c3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e70 expref: 320 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.104531:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032e8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993226880:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.104563:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02645d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.104595:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02645d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.104595:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.104598:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000bbb00b33/0xd5401cbabc021ba1 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7c3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1e70 expref: 320 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.104640:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.104645:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02645d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.104662:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.104672:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.104677:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026472 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.104685:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026472 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.104707:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513bc0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+687:3627:x1870152993225216:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3447us (3584us total) trans 4294976256 rc 0/0 00010000:00010000:0.0:1783520936.104710:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.104714:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44756 00000100:00100000:2.0:1783520936.104743:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993226880 00000100:00100000:2.0:1783520936.104747:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993226880 00000100:00100000:0.0:1783520936.104748:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513bc1880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+319:2388:x1870153002455296:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 396us (694us total) trans 0 rc 0/0 00000004:00080000:1.0:1783520936.104754:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513bc1c00/2, rc 0, transno 4294976256 00000100:00100000:0.0:1783520936.104763:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36347 00000100:00100000:1.0:1783520936.104771:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993225216:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.104793:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:2.0:1783520936.104797:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44761 00010000:00010000:3.0:1783520936.104802:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02646b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.104814:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea200 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520936.104819:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032eb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+687:3627:x1870152993226880:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.104820:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02646b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.104824:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026472 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.104869:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026472 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.104874:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993227008:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.104886:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026472 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.104897:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026472 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.104917:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02646b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.104936:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.104950:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc1500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993226112:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.104957:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02646b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.104966:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc02646b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.104982:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026479 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.105004:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.105009:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.105015:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.105022:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993227008 00010000:00010000:0.0:1783520936.105023:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.105029:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993227008 00000100:00100000:2.0:1783520936.105080:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44762 00010000:00010000:3.0:1783520936.105102:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:2.0:1783520936.105104:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516649500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+689:3625:x1870152993227008:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.105108:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026472 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.105117:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.105121:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026472 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.105125:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1995 count 1 00000100:00100000:0.0:1783520936.105138:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351720b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+689:3626:x1870152993224960:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4356us (4551us total) trans 4294976257 rc 0/0 00010000:00010000:2.0:1783520936.105147:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026495 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7cb:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.105147:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44753 00010000:00010000:2.0:1783520936.105167:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000edd5a45d/0xd5401cbabc021be0 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ee0 expref: 318 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.105176:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7cb:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.105181:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7cb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.105187:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026495 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.105194:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351720b100/2, rc 0, transno 4294976257 00000100:00100000:1.0:1783520936.105202:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.105217:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351720b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993224960:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.105237:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002455424 00000100:00100000:2.0:1783520936.105243:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002455424 00000100:00100000:0.0:1783520936.105263:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc3100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993226496:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.105271:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a794a83f/0xd5401cbabc021bc4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ea8 expref: 319 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.105292:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026480 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.105313:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36348 00000100:00100000:1.0:1783520936.105331:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.105344:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032eb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+319:2387:x1870153002455424:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.105379:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026472 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.105391:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.105402:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a794a83f/0xd5401cbabc021bc4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7c7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ea8 expref: 319 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.105404:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026472 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.105423:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.105432:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032ea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993227136:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.105454:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a794a83f/0xd5401cbabc021bc4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7c7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ea8 expref: 319 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.105464:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002455552 00000100:00100000:0.0:1783520936.105470:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.105471:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002455552 00010000:00010000:2.0:1783520936.105477:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a794a83f/0xd5401cbabc021bc4 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7c7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ea8 expref: 319 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.105500:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.105504:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.105509:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ae76f4cd/0xd5401cbabc021bcb lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1eb6 expref: 319 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.105513:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026479 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.105529:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993227136 00000100:00100000:0.0:1783520936.105538:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993227136 00010000:00010000:3.0:1783520936.105558:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026472 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.105563:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026479 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.105571:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44763 00000100:00100000:1.0:1783520936.105582:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002455680 00000100:00100000:1.0:1783520936.105595:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002455680 00000100:00100000:0.0:1783520936.105597:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513bc3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+689:3626:x1870152993227136:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.105613:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.105619:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.105620:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d42d5e00/0xd5401cbabc022b45 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3235 expref: 320 pid: 8409 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.105622:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1996 count 1 00010000:00010000:3.0:1783520936.105624:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.105634:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.105648:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02649c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7cc:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.105672:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36349 00010000:00010000:0.0:1783520936.105680:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a8803b2e/0xd5401cbabc021be7 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1eee expref: 320 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.105682:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032eb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+320:2387:x1870153002455424:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 345us (486us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.105692:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36350 00000100:00100000:3.0:1783520936.105702:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032e8700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+320:2389:x1870153002455552:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.105732:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36348 00000100:00100000:1.0:1783520936.105733:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638585450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+320:2390:x1870153002455680:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.105735:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7cc:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.105744:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.105756:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7cc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.105762:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.105764:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ae76f4cd/0xd5401cbabc021bcb lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7c8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1eb6 expref: 319 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.105767:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02649c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.105768:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026479 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.105778:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d42d5e00/0xd5401cbabc022b45 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7c6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3235 expref: 319 pid: 8409 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.105833:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026479 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.105835:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ae76f4cd/0xd5401cbabc021bcb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7c8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1eb6 expref: 319 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.105849:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d42d5e00/0xd5401cbabc022b45 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7c6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3235 expref: 319 pid: 8409 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.105859:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a8803b2e/0xd5401cbabc021be7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1eee expref: 319 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.105896:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026479 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.105903:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.105929:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.105929:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d42d5e00/0xd5401cbabc022b45 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7c6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3235 expref: 319 pid: 8409 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.105930:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a8803b2e/0xd5401cbabc021be7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1eee expref: 319 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.105937:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026479 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.105952:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026487 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.106001:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc1500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993227264:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.106007:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.106021:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.106027:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.106030:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026487 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7c8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.106031:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.106032:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026480 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.106046:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026479 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.106061:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435192190c0 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.106077:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026480 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.106080:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.106086:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.106090:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026479 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.106107:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026479 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.106118:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.106126:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032e8700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+318:2389:x1870153002455552:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 436us (807us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.106129:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026479 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.106156:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638585450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+318:2390:x1870153002455680:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 435us (739us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.106164:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36349 00010000:00010000:2.0:1783520936.106194:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026479 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.106195:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36350 00000100:00100000:0.0:1783520936.106234:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.106241:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.106258:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.106265:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.106265:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02645d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.106284:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.106290:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02645d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.106294:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002455808 00010000:00010000:2.0:1783520936.106296:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02645d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.106303:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002455808 00010000:00010000:2.0:1783520936.106311:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02645d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.106323:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02645d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.106327:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f590c184/0xd5401cbabc021bd2 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ec4 expref: 317 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.106443:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.106453:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.106458:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36351 00010000:00010000:2.0:1783520936.106463:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.106471:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513bc0700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+317:2388:x1870153002455808:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.106474:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026480 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.106480:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.106488:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f590c184/0xd5401cbabc021bd2 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7c9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ec4 expref: 317 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.106517:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026480 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.106523:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f590c184/0xd5401cbabc021bd2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7c9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ec4 expref: 317 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.106550:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026480 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.106551:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+691:3628:x1870152993224832:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5623us (5854us total) trans 4294976258 rc 0/0 00010000:00010000:1.0:1783520936.106562:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.106579:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7c9:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.106597:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44754 00010000:00010000:1.0:1783520936.106598:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02648e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.106600:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026480 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.106629:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511adaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+690:3626:x1870152993225984:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4323us (4515us total) trans 4294976259 rc 0/0 00010000:00010000:1.0:1783520936.106641:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02648e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7c9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.106657:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44758 00000004:00080000:3.0:1783520936.106658:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351720bb80/2, rc 0, transno 4294976258 00010000:00010000:1.0:1783520936.106673:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.106685:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351720bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993224832:0@lo:6:osp-syn-1-0.0 00000004:00080000:0.0:1783520936.106696:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943511adad80/2, rc 0, transno 4294976259 00000100:00100000:0.0:1783520936.106709:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511adad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993225984:0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.106710:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1994 count 1 00000100:00100000:1.0:1783520936.106729:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513bc0700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+317:2388:x1870153002455808:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 256us (496us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.106730:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0264a3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7ca:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.106748:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.106766:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36351 00010000:00010000:3.0:1783520936.106775:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.106778:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000da7a1b89/0xd5401cbabc021bd9 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ed2 expref: 317 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.106782:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026480 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.106809:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993227392:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.106822:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.106830:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026480 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.106869:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513bc1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+690:3628:x1870152993225856:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4618us (4802us total) trans 4294976260 rc 0/0 00010000:00010000:3.0:1783520936.106875:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026480 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.106881:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.106884:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ca:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.106902:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ca:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.106909:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44757 00010000:00010000:2.0:1783520936.106913:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0264a3 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.106923:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1783520936.106938:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993227392 00000100:00100000:1.0:1783520936.106942:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993227520:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.106954:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993227392 00010000:00010000:3.0:1783520936.106959:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026480 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.106967:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026480 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.106986:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000da7a1b89/0xd5401cbabc021bd9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ed2 expref: 316 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.106997:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.107003:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.107008:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000da7a1b89/0xd5401cbabc021bd9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ed2 expref: 316 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.107011:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.107012:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.107013:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44764 00000100:00100000:0.0:1783520936.107035:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032eb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+690:3627:x1870152993227392:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.107040:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516649180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993227648:192.168.204.33@tcp:104: 00002000:00080000:0.0:1783520936.107053:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1997 count 1 00000004:00080000:3.0:1783520936.107057:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943511adbb80/2, rc 0, transno 4294976260 00010000:00010000:0.0:1783520936.107071:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0264aa lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7cd:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.107073:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511adbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993225856:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.107095:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993227520 00000100:00100000:1.0:1783520936.107104:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993227520 00010000:00010000:0.0:1783520936.107105:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000042fd495f/0xd5401cbabc021bf5 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1efc expref: 316 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.107117:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7cd:0x0].0x0 since=38s v=(0 0) 00000100:00100000:2.0:1783520936.107119:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.107122:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7cd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.107129:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0264aa lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.107145:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44765 00000100:00100000:1.0:1783520936.107157:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032e8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+691:3627:x1870152993227520:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.107167:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000042fd495f/0xd5401cbabc021bf5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1efc expref: 316 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.107178:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993227776:0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.107178:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1998 count 1 00010000:00010000:0.0:1783520936.107187:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000042fd495f/0xd5401cbabc021bf5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1efc expref: 316 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.107201:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc2d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993227904:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.107203:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0264b1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7ce:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.107219:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000edd5a45d/0xd5401cbabc021be0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ee0 expref: 316 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.107232:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.107244:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001176ed4b/0xd5401cbabc022a34 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3124 expref: 316 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.107254:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x7ce:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.107255:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000edd5a45d/0xd5401cbabc021be0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ee0 expref: 316 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.107258:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ce:0x0].0x0 since=0s v=(1 0) 00000100:00100000:0.0:1783520936.107258:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514219840 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.107271:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0264b1 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.107278:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993227776 00000100:00100000:3.0:1783520936.107285:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993228032:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.107288:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993227776 00010000:00010000:0.0:1783520936.107302:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f590c184/0xd5401cbabc021bd2 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ec4 expref: 316 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.107313:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001176ed4b/0xd5401cbabc022a34 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3124 expref: 316 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.107325:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc3800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993226752:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.107334:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001176ed4b/0xd5401cbabc022a34 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3124 expref: 315 pid: 20140 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.107336:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.107337:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44766 00010000:00010000:0.0:1783520936.107347:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02648e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.107389:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02648e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.107390:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351664a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+692:3625:x1870152993227776:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.107413:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02648e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.107420:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:1999 count 1 00000100:00100000:1.0:1783520936.107421:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032eaa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993228160:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.107436:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0264b8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7cf:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.107446:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02648e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.107454:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009f491ddd/0xd5401cbabc021c03 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f18 expref: 315 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.107474:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7cf:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.107474:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02648e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.107480:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7cf:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.107487:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470240 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.107500:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0264b8 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.107520:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.107531:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.107539:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009f491ddd/0xd5401cbabc021c03 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f18 expref: 315 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.107541:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02648e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.107594:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.107600:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02648e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.107602:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009f491ddd/0xd5401cbabc021c03 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f18 expref: 315 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.107616:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02648e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.107653:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+693:3628:x1870152993226368:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4372us (4807us total) trans 4294976261 rc 0/0 00010000:00010000:3.0:1783520936.107659:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02648e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.107661:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516649c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993228288:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.107669:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44760 00010000:00010000:3.0:1783520936.107688:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc02648e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.107723:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.107727:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.107735:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.107736:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d83c0 going to sleep for 11 seconds 00000004:00080000:0.0:1783520936.107739:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943511ad8a80/2, rc 0, transno 4294976261 00000100:00100000:0.0:1783520936.107768:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ad8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993226368:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.107855:0:8408:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ae76f4cd/0xd5401cbabc021bcb lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1eb6 expref: 315 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.107860:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993228416:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.107881:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc1500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993227264:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.107882:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351664ad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993226624:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.107908:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.107945:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02649c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.107946:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026487 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.107976:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.107985:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993228416 00000100:00100000:0.0:1783520936.107996:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993228416 00010000:00010000:2.0:1783520936.108001:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026487 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.108016:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026487 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.108025:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026487 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.108033:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026487 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.108035:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44767 00000100:00100000:0.0:1783520936.108047:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513bc3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+693:3626:x1870152993228416:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.108054:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00002000:00080000:0.0:1783520936.108061:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2000 count 1 00010000:00010000:0.0:1783520936.108075:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0264bf lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7d0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.108078:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.108092:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026487 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.108093:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002455936 00000100:00100000:1.0:1783520936.108102:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002455936 00010000:00010000:0.0:1783520936.108102:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e1fbda0a/0xd5401cbabc021fe5 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f26 expref: 314 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.108117:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.108121:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d0:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.108125:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026487 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.108128:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.108139:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a8803b2e/0xd5401cbabc021be7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1eee expref: 315 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.108151:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0264bf lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.108164:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+694:3625:x1870152993226240:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5400us (5501us total) trans 4294976262 rc 0/0 00010000:00010000:3.0:1783520936.108169:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026487 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.108213:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44759 00000004:00080000:0.0:1783520936.108238:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943511ad9880/2, rc 0, transno 4294976262 00010000:00010000:3.0:1783520936.108250:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026487 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.108256:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ad9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993226240:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.108275:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026487 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.108282:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36352 00000100:00100000:1.0:1783520936.108304:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee5500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+315:2387:x1870153002455936:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.108313:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.108316:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.108317:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.108320:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.108327:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a8803b2e/0xd5401cbabc021be7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7cc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1eee expref: 315 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.108331:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:1.0:1783520936.108350:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a8803b2e/0xd5401cbabc021be7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7cc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1eee expref: 315 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.108354:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993228544:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.108401:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a8803b2e/0xd5401cbabc021be7 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7cc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1eee expref: 315 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.108419:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7cc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.108422:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7cc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.108431:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02649c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.108433:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.108443:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02649c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.108470:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.108483:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993228544 00000100:00100000:2.0:1783520936.108501:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993228544 00000100:00100000:1.0:1783520936.108525:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee5500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+314:2387:x1870153002455936:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 229us (560us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.108534:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e1fbda0a/0xd5401cbabc021fe5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f26 expref: 314 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.108546:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36352 00000100:00100000:2.0:1783520936.108557:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44768 00010000:00010000:0.0:1783520936.108560:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e1fbda0a/0xd5401cbabc021fe5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f26 expref: 314 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.108572:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513bc2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+694:3628:x1870152993228544:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.108581:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc0000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993228672:192.168.204.33@tcp:104: 00002000:00080000:2.0:1783520936.108588:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2001 count 1 00010000:00010000:2.0:1783520936.108609:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0264c6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7d1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.108620:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02649c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.108629:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c4e1a612/0xd5401cbabc021c11 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f34 expref: 313 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.108642:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02649c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.108645:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514219780 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.108655:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d1:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.108675:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.108681:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02649c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.108681:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0264c6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.108745:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c4e1a612/0xd5401cbabc021c11 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f34 expref: 313 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.108752:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c4e1a612/0xd5401cbabc021c11 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f34 expref: 313 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.108759:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351664ad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993228800:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.108787:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57d80 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.108815:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02649c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.108935:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.108964:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.108972:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02649c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.108982:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.108987:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02649c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.108996:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02649c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.109011:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02649c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.109024:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02649c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.109046:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513bc3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+695:3626:x1870152993227136:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3463us (3575us total) trans 4294976263 rc 0/0 00010000:00010000:2.0:1783520936.109047:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.109054:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.109064:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.109065:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44763 00000004:00080000:1.0:1783520936.109093:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435032ea300/2, rc 0, transno 4294976263 00000100:00100000:1.0:1783520936.109116:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032ea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993227136:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.109214:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1783520936.109230:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993228928:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.109272:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.109320:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993228928 00000100:00100000:0.0:1783520936.109327:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993228928 00000100:00100000:0.0:1783520936.109384:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44769 00000100:00100000:0.0:1783520936.109396:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+695:3627:x1870152993228928:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.109413:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2002 count 1 00010000:00010000:0.0:1783520936.109430:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0264cd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7d2:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.109444:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000052a5e7aa/0xd5401cbabc021c18 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f42 expref: 313 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.109452:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d2:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.109455:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.109459:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0264cd lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.109480:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000052a5e7aa/0xd5401cbabc021c18 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f42 expref: 313 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.109504:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000052a5e7aa/0xd5401cbabc021c18 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f42 expref: 313 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.109517:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc1500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993229056:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.109562:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c016cc0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.110068:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516649180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993227648:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.110111:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc2d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993227904:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.110115:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0264a3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.110119:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993228032:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.110148:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0264aa lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.110161:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026495 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.110161:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.110186:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516649c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993228288:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.110216:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.110241:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032eaa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993228160:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.110272:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.110272:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0264b1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.110276:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0264b8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.110309:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.110345:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002456064 00000100:00100000:0.0:1783520936.110352:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002456064 00000100:00100000:1.0:1783520936.110440:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.110467:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000da7a1b89/0xd5401cbabc021bd9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ed2 expref: 314 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.110506:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002456192 00000100:00100000:2.0:1783520936.110510:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002456320 00000100:00100000:3.0:1783520936.110516:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002456192 00000100:00100000:0.0:1783520936.110517:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002456448 00000100:00100000:2.0:1783520936.110520:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002456320 00000100:00100000:0.0:1783520936.110526:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002456448 00010000:00010000:2.0:1783520936.110556:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000edd5a45d/0xd5401cbabc021be0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ee0 expref: 317 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.110557:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001176ed4b/0xd5401cbabc022a34 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3124 expref: 317 pid: 20140 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.110596:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.110608:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002456576 00000100:00100000:0.0:1783520936.110617:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002456576 00000100:00100000:2.0:1783520936.110622:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002456704 00000100:00100000:2.0:1783520936.110629:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002456704 00010000:00010000:3.0:1783520936.110648:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000042fd495f/0xd5401cbabc021bf5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1efc expref: 319 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.110660:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009f491ddd/0xd5401cbabc021c03 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f18 expref: 319 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.110666:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.110668:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e1fbda0a/0xd5401cbabc021fe5 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f26 expref: 319 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.110715:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002456832 00000100:00100000:3.0:1783520936.110723:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002456832 00000100:00100000:0.0:1783520936.110740:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36353 00010000:00010000:3.0:1783520936.110753:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c4e1a612/0xd5401cbabc021c11 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f34 expref: 320 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.110757:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36354 00000100:00100000:0.0:1783520936.110763:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032e9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+320:2390:x1870153002456064:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.110785:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+320:2390:x1870153002456448:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.110791:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.110800:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36355 00010000:00010000:2.0:1783520936.110806:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.110806:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000da7a1b89/0xd5401cbabc021bd9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ca:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ed2 expref: 320 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.110820:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001176ed4b/0xd5401cbabc022a34 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7ce:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3124 expref: 320 pid: 20140 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.110822:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36356 00000100:00100000:1.0:1783520936.110823:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+320:2389:x1870153002456320:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.110852:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001176ed4b/0xd5401cbabc022a34 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7ce:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3124 expref: 320 pid: 20140 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.110856:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.110873:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000edd5a45d/0xd5401cbabc021be0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7cb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ee0 expref: 320 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.110885:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001176ed4b/0xd5401cbabc022a34 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7ce:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3124 expref: 320 pid: 20140 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.110885:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000da7a1b89/0xd5401cbabc021bd9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7ca:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ed2 expref: 320 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.110910:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000edd5a45d/0xd5401cbabc021be0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7cb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ee0 expref: 320 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.110926:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ce:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.110931:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000da7a1b89/0xd5401cbabc021bd9 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7ca:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ed2 expref: 319 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.110933:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ce:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.110943:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000edd5a45d/0xd5401cbabc021be0 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7cb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ee0 expref: 319 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.110944:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0264b1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.110957:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ca:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.110978:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ca:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.110993:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0264b1 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.110995:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7cb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.111007:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7cb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.111010:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0264a3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.111021:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026495 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.111036:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032e9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+317:2388:x1870153002456192:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.111049:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.111065:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0264a3 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.111070:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.111076:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026495 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.111085:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000042fd495f/0xd5401cbabc021bf5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7cd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1efc expref: 317 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.111101:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+317:2390:x1870153002456448:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 323us (790us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.111130:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.111141:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36354 00010000:00010000:3.0:1783520936.111148:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000042fd495f/0xd5401cbabc021bf5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7cd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1efc expref: 317 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.111160:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.111172:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000042fd495f/0xd5401cbabc021bf5 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7cd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1efc expref: 316 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.111174:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032e9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+316:2390:x1870153002456064:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 416us (1011us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.111184:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36357 00000100:00100000:1.0:1783520936.111204:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+315:2389:x1870153002456320:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 383us (927us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.111205:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7cd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.111213:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7cd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.111218:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0264aa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.111223:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0264aa lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.111240:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.111279:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638587050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+315:2389:x1870153002456576:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.111279:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36353 00000100:00100000:3.0:1783520936.111280:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032e9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+315:2388:x1870153002456192:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 424us (1048us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.111297:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36355 00000100:00100000:3.0:1783520936.111313:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36356 00010000:00010000:2.0:1783520936.111318:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.111323:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36358 00010000:00010000:2.0:1783520936.111331:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009f491ddd/0xd5401cbabc021c03 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7cf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f18 expref: 312 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.111339:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36359 00000100:00100000:0.0:1783520936.111350:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+312:2390:x1870153002456704:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.111384:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+312:2390:x1870153002456832:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.111395:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009f491ddd/0xd5401cbabc021c03 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7cf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f18 expref: 312 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.111408:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.111413:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.111423:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e1fbda0a/0xd5401cbabc021fe5 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7d0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f26 expref: 312 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.111431:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c4e1a612/0xd5401cbabc021c11 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7d1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f34 expref: 312 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.111432:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.111432:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009f491ddd/0xd5401cbabc021c03 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7cf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f18 expref: 312 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.111507:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c4e1a612/0xd5401cbabc021c11 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7d1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f34 expref: 311 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.111508:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7cf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.111513:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e1fbda0a/0xd5401cbabc021fe5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7d0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f26 expref: 311 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.111527:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7cf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.111548:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0264b8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.111566:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0264b8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.111571:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.111577:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.111579:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.111582:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026495 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.111592:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.111597:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.111602:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0264c6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.111620:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0264bf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.111624:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0264c6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7d1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.111640:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638587050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+311:2389:x1870153002456576:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 432us (1198us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.111669:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026495 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.111671:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0264bf lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7d0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.111689:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36357 00010000:00010000:3.0:1783520936.111730:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026495 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.111743:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.111745:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.111749:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026495 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.111757:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002456960 00000100:00100000:2.0:1783520936.111764:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002456960 00010000:00010000:3.0:1783520936.111774:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.111781:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+311:2390:x1870153002456832:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 427us (1127us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.111789:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000052a5e7aa/0xd5401cbabc021c18 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f42 expref: 311 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.111793:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+311:2390:x1870153002456704:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 453us (1204us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.111794:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520936.111800:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36359 00010000:00010000:3.0:1783520936.111812:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026495 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.111854:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.111862:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026495 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.111872:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026495 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.111881:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36358 00010000:00010000:3.0:1783520936.111909:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026495 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.111918:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026495 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.111928:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36360 00000100:00100000:2.0:1783520936.111939:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee7100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+309:2387:x1870153002456960:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.111952:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.111958:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000052a5e7aa/0xd5401cbabc021c18 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7d2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f42 expref: 309 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.111973:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000052a5e7aa/0xd5401cbabc021c18 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7d2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f42 expref: 309 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.111974:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0264aa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.111977:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc1500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993229056:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.111983:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.112011:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0264cd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7d2:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.112013:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000052a5e7aa/0xd5401cbabc021c18 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7d2:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f42 expref: 309 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.112039:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.112046:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.112068:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.112068:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0264aa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.112075:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.112084:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0264cd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.112087:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0264aa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.112092:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0264cd lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.112103:0:8410:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e1fbda0a/0xd5401cbabc021fe5 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f26 expref: 308 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.112110:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0264aa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.112135:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.112158:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc0000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993228672:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.112173:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee7100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+307:2387:x1870153002456960:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 235us (755us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.112174:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.112178:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.112178:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0264bf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.112190:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0264aa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.112211:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36360 00010000:00010000:1.0:1783520936.112213:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0264bf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.112220:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.112229:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0264aa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.112236:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0264bf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.112244:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0264aa lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.112249:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0264bf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.112253:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0264b8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.112281:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0264aa lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.112309:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0264b8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.112331:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032eb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+697:3627:x1870152993227392:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5291us (5452us total) trans 4294976265 rc 0/0 00010000:00010000:3.0:1783520936.112342:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0264aa lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.112342:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0264bf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.112397:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0264b8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.112398:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.112403:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520936.112415:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44764 00010000:00010000:2.0:1783520936.112417:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0264b8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.112432:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.112443:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.112449:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0264b8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.112454:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.112458:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0264b8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.112465:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0264b8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.112482:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0264b8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.112486:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0264bf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.112510:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943508ee7b80/2, rc 0, transno 4294976265 00000100:00100000:0.0:1783520936.112533:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993227392:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.112539:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbebe00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520936.112544:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513bc3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+697:3626:x1870152993228416:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4490us (4634us total) trans 4294976266 rc 0/0 00010000:00010000:2.0:1783520936.112556:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0264b8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.112558:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0264bf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.112582:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44767 00010000:00010000:3.0:1783520936.112609:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0264bf lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.112633:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0264bf lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.112639:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0264bf lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.112649:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520936.112653:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513bc3800/2, rc 0, transno 4294976266 00000100:00100000:0.0:1783520936.112655:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993229184:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.112659:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.112664:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.112667:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.112667:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.112670:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993228416:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.112678:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.112692:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.112695:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0264cd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.112703:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0264cd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.112710:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0264cd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.112726:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.112737:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0264cd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.112768:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.112785:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.112792:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0264cd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.112799:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.112802:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0264cd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.112824:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351664a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+698:3625:x1870152993227776:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5469us (5599us total) trans 4294976267 rc 0/0 00000100:00100000:2.0:1783520936.112833:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44766 00010000:00010000:3.0:1783520936.112845:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0264cd lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.112861:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993229312:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.112864:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0264cd lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.112866:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993229184 00000100:00100000:0.0:1783520936.112875:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993229184 00010000:00010000:3.0:1783520936.112880:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc0264cd lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.112885:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943508ee5180/2, rc 0, transno 4294976267 00000100:00100000:1.0:1783520936.112915:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.112920:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.112923:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993227776:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.112925:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.112931:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.112961:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993229312 00000100:00100000:3.0:1783520936.112966:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993229312 00010000:00010000:2.0:1783520936.113004:0:12734:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c4e1a612/0xd5401cbabc021c11 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f34 expref: 306 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.113021:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+699:3627:x1870152993228928:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3624us (3744us total) trans 4294976268 rc 0/0 00000100:00100000:3.0:1783520936.113022:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44770 00000100:00100000:2.0:1783520936.113022:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351664ad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993228800:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.113046:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513bc0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+699:3625:x1870152993229184:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.113049:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993229440:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.113057:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0264c6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.113060:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44769 00010000:00010000:2.0:1783520936.113076:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0264c6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.113082:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0264c6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.113087:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2003 count 1 00000100:00100000:1.0:1783520936.113098:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44771 00010000:00010000:2.0:1783520936.113108:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0264c6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.113116:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+698:3626:x1870152993229312:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.113117:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0264d4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7d3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.113139:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.113145:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0264c6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.113152:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2004 count 1 00010000:00010000:3.0:1783520936.113180:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ec1face9/0xd5401cbabc021c26 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f50 expref: 305 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.113184:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513bc1f80/2, rc 0, transno 4294976268 00010000:00010000:1.0:1783520936.113186:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0264db lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7d4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.113188:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.113192:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d3:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.113204:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d3:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.113211:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993228928:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.113212:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.113214:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0264d4 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.113228:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0264c6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.113234:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000466d4539/0xd5401cbabc021c2d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f5e expref: 305 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.113273:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.113276:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d4:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.113279:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0264c6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.113284:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ec1face9/0xd5401cbabc021c26 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f50 expref: 305 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.113288:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d4:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.113310:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993229696:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.113312:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0264db lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.113327:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0264c6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.113355:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ec1face9/0xd5401cbabc021c26 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f50 expref: 305 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.113412:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.113417:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0264c6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.113424:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0264c6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.113444:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000466d4539/0xd5401cbabc021c2d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f5e expref: 305 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.113453:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.113462:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.113470:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000466d4539/0xd5401cbabc021c2d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f5e expref: 305 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.113472:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.113488:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993229440 00000100:00100000:2.0:1783520936.113505:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993229440 00000100:00100000:1.0:1783520936.113511:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b21180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993229824:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.113515:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0264b1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.113527:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993229696 00000100:00100000:2.0:1783520936.113536:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993229696 00000100:00100000:2.0:1783520936.113561:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44772 00010000:00010000:0.0:1783520936.113571:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0264b1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.113578:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513bc0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+700:3627:x1870152993229440:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.113580:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993229568:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.113589:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470900 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.113604:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0264b1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.113606:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2005 count 1 00010000:00010000:0.0:1783520936.113616:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0264b1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.113628:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0264e2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7d5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.113633:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6c00 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.113642:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.113649:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.113649:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000873a36bb/0xd5401cbabc021c34 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f6c expref: 305 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.113656:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0264b1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.113690:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d5:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.113695:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbbe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.113698:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.113703:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0264b1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.113706:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0264e2 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.113726:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44773 00000100:00100000:0.0:1783520936.113739:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032e8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+701:3627:x1870152993227520:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6582us (6740us total) trans 4294976269 rc 0/0 00000100:00100000:1.0:1783520936.113751:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513bc0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+701:3628:x1870152993229696:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.113761:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000873a36bb/0xd5401cbabc021c34 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f6c expref: 305 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.113765:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44765 00010000:00010000:2.0:1783520936.113778:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000873a36bb/0xd5401cbabc021c34 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f6c expref: 305 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.113782:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2006 count 1 00000100:00100000:2.0:1783520936.113802:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351664ad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993229952:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.113806:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0264e9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7d6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.113820:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943508ee6d80/2, rc 0, transno 4294976269 00000100:00100000:2.0:1783520936.113840:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435166d83c0 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.113844:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0264b1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.113850:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001cd9e5c0/0xd5401cbabc022923 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3013 expref: 305 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.113851:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993227520:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.113877:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x7d6:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.113880:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.113883:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0264e9 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.113917:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001cd9e5c0/0xd5401cbabc022923 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3013 expref: 305 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.113927:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0264b1 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.113935:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513bc2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+701:3628:x1870152993228544:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5364us (5505us total) trans 4294976270 rc 0/0 00010000:00010000:1.0:1783520936.113937:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001cd9e5c0/0xd5401cbabc022923 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3013 expref: 305 pid: 9151 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.113965:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0264b1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.113981:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44768 00000100:00100000:1.0:1783520936.113994:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b20e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993230208:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.114008:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.114015:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.114020:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.114023:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1783520936.114037:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993230080:0@lo:6:osp-syn-1-0.0 00000004:00080000:2.0:1783520936.114051:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513bc3b80/2, rc 0, transno 4294976270 00000100:00100000:2.0:1783520936.114071:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993228544:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.114088:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.114132:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1783520936.114148:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516649c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993230336:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.114159:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470b40 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.114178:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.114191:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993230080 00000100:00100000:0.0:1783520936.114196:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993230080 00000100:00100000:2.0:1783520936.114249:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993230336 00000100:00100000:0.0:1783520936.114255:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44774 00000100:00100000:2.0:1783520936.114256:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993230336 00000100:00100000:3.0:1783520936.114265:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516649500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+702:3625:x1870152993227008:12345-0@lo:6:osp-syn-1-0.0 Request processed in 9169us (9334us total) trans 4294976264 rc 0/0 00000100:00100000:0.0:1783520936.114276:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b23100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+702:3626:x1870152993230080:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.114286:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44762 00002000:00080000:0.0:1783520936.114295:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2007 count 1 00010000:00010000:0.0:1783520936.114313:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0264f0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7d7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.114326:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e9d0730c/0xd5401cbabc021c42 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f88 expref: 305 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.114334:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44775 00010000:00010000:0.0:1783520936.114336:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d7:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.114341:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d7:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.114346:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516649180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+701:3625:x1870152993230336:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.114349:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0264f0 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.114408:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435032e8380/2, rc 0, transno 4294976264 00010000:00010000:0.0:1783520936.114425:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e9d0730c/0xd5401cbabc021c42 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f88 expref: 305 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.114432:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2008 count 1 00010000:00010000:0.0:1783520936.114438:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e9d0730c/0xd5401cbabc021c42 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f88 expref: 305 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.114446:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0264f7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7d8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.114467:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032e9500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993230464:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.114469:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000bc40a014/0xd5401cbabc021c50 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f96 expref: 305 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.114487:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d8:0x0].0x0 since=38s v=(0 0) 00000100:00100000:3.0:1783520936.114504:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993227008:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.114507:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.114517:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0264f7 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.114544:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000bc40a014/0xd5401cbabc021c50 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f96 expref: 305 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.114547:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514219780 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.114564:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000bc40a014/0xd5401cbabc021c50 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f96 expref: 305 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.114574:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516648380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993230592:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.114599:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993230720:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.114641:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b21180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993229824:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.114652:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57d80 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.114671:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.114684:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993230720 00010000:00010000:1.0:1783520936.114685:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0264db lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.114691:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993230720 00000100:00100000:2.0:1783520936.114743:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44776 00000100:00100000:2.0:1783520936.114760:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032eaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+702:3627:x1870152993230720:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.114762:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0264a3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.114774:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0264a3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.114777:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2009 count 1 00010000:00010000:0.0:1783520936.114788:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0264a3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.114793:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0264fe lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7d9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.114798:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0264a3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.114806:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000cca6eef3/0xd5401cbabc021c57 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fa4 expref: 305 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.114812:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d9:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.114815:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.114817:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0264fe lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.114819:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.114843:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000cca6eef3/0xd5401cbabc021c57 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fa4 expref: 305 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.114860:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000cca6eef3/0xd5401cbabc021c57 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fa4 expref: 305 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.114872:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516649880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993230848:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.114919:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.114932:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032eb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+703:3627:x1870152993226880:12345-0@lo:6:osp-syn-1-0.0 Request processed in 10124us (10306us total) trans 4294976271 rc 0/0 00000100:00100000:0.0:1783520936.114940:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44761 00010000:00010000:3.0:1783520936.114946:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.114956:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0264a3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.114968:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.114975:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0264a3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.114990:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0264a3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.114996:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435032e8e00/2, rc 0, transno 4294976271 00010000:00010000:3.0:1783520936.115006:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0264a3 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.115018:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0264a3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.115025:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032e8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993226880:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.115027:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993229568:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.115054:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.115084:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0264d4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.115105:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002457088 00000100:00100000:1.0:1783520936.115113:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002457088 00010000:00010000:3.0:1783520936.115115:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.115122:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.115128:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.115139:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350626e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993230976:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.115146:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000466d4539/0xd5401cbabc021c2d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f5e expref: 306 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.115185:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.115192:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36361 00000100:00100000:1.0:1783520936.115209:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b21180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+306:2388:x1870153002457088:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.115221:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.115226:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993230976 00010000:00010000:1.0:1783520936.115229:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000466d4539/0xd5401cbabc021c2d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7d4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f5e expref: 306 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.115233:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993230976 00010000:00010000:1.0:1783520936.115246:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000466d4539/0xd5401cbabc021c2d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7d4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f5e expref: 306 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.115264:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000466d4539/0xd5401cbabc021c2d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7d4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f5e expref: 306 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.115272:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44777 00000100:00100000:3.0:1783520936.115286:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351664bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+703:3628:x1870152993230976:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.115286:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.115294:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.115299:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0264db lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.115305:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0264db lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.115306:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2010 count 1 00010000:00010000:3.0:1783520936.115326:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026505 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7da:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.115335:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.115341:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f21ad4cb/0xd5401cbabc021c5e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fb2 expref: 305 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.115348:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7da:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.115350:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7da:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.115355:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026505 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.115414:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f21ad4cb/0xd5401cbabc021c5e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fb2 expref: 305 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.115420:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0264db lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.115440:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f21ad4cb/0xd5401cbabc021c5e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fb2 expref: 305 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.115451:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b21180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+305:2388:x1870153002457088:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 240us (393us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.115481:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36361 00010000:00010000:0.0:1783520936.115503:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0264db lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.115510:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993231104:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.115530:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0264db lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.115541:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0264db lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.115567:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.115581:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.115590:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0264db lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.115600:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.115606:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0264db lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.115618:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0264db lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.115627:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6240 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.115642:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0264db lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.115652:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0264db lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.115672:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.115675:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.115677:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.115695:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+704:3626:x1870152993229312:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2584us (2785us total) trans 4294976272 rc 0/0 00000100:00100000:0.0:1783520936.115707:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44771 00000004:00080000:1.0:1783520936.115721:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513bc1880/2, rc 0, transno 4294976272 00000100:00100000:1.0:1783520936.115738:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993229312:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.115809:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1783520936.115822:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516649f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993231232:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.115855:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.115893:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993231232 00000100:00100000:2.0:1783520936.115900:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993231232 00000100:00100000:1.0:1783520936.115919:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.115952:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44778 00000100:00100000:2.0:1783520936.115967:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b20a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+704:3626:x1870152993231232:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.115972:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002457216 00000100:00100000:0.0:1783520936.115979:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002457216 00002000:00080000:2.0:1783520936.115984:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2011 count 1 00010000:00010000:2.0:1783520936.116002:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02650c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7db:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.116008:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ec1face9/0xd5401cbabc021c26 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f50 expref: 305 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.116021:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005679b630/0xd5401cbabc021c65 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fc0 expref: 305 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.116030:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7db:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.116034:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7db:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.116039:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02650c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.116048:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36362 00000100:00100000:0.0:1783520936.116061:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b22300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+305:2387:x1870153002457216:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.116068:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005679b630/0xd5401cbabc021c65 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fc0 expref: 305 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.116086:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.116093:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351664ad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993229952:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.116095:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.116096:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005679b630/0xd5401cbabc021c65 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fc0 expref: 305 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.116107:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ec1face9/0xd5401cbabc021c26 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7d3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f50 expref: 305 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.116118:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351664aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993231360:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.116135:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.116142:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ec1face9/0xd5401cbabc021c26 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7d3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f50 expref: 305 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.116157:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0264e2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.116170:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ec1face9/0xd5401cbabc021c26 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7d3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f50 expref: 305 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.116179:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.116188:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.116193:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.116199:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0264d4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.116209:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0264d4 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.116234:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.116249:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0264d4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.116260:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0264d4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.116267:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b22300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+304:2387:x1870153002457216:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 206us (344us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.116272:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0264d4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.116281:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36362 00010000:00010000:2.0:1783520936.116284:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0264d4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.116310:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.116313:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002457344 00000100:00100000:1.0:1783520936.116314:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002457472 00000100:00100000:1.0:1783520936.116321:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002457472 00010000:00010000:2.0:1783520936.116323:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520936.116323:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002457344 00010000:00010000:2.0:1783520936.116331:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0264d4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.116339:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.116346:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0264d4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.116347:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000873a36bb/0xd5401cbabc021c34 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f6c expref: 305 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.116356:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001cd9e5c0/0xd5401cbabc022923 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x7d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3013 expref: 305 pid: 9151 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.116438:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0264d4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.116455:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0264d4 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.116464:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc0264d4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.116472:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36363 00000100:00100000:0.0:1783520936.116501:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638587450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+305:2387:x1870153002457472:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.116506:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.116514:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36364 00010000:00010000:2.0:1783520936.116516:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.116520:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.116526:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.116532:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000873a36bb/0xd5401cbabc021c34 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7d5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f6c expref: 305 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.116540:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b22680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+305:2388:x1870153002457344:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.116566:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.116571:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000873a36bb/0xd5401cbabc021c34 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7d5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f6c expref: 305 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.116574:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.116581:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001cd9e5c0/0xd5401cbabc022923 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x7d6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3013 expref: 305 pid: 9151 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.116620:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001cd9e5c0/0xd5401cbabc022923 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7d6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3013 expref: 305 pid: 9151 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.116628:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000873a36bb/0xd5401cbabc021c34 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7d5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f6c expref: 305 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.116648:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.116654:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.116659:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.116660:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0264e9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.116661:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513bc0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+705:3625:x1870152993229184:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3617us (3943us total) trans 4294976273 rc 0/0 00010000:00010000:0.0:1783520936.116685:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d5:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.116691:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.116704:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44770 00010000:00010000:0.0:1783520936.116709:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0264e2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.116722:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0264e9 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7d6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.116728:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0264e2 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.116753:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.116760:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516648380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993230592:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.116760:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.116772:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.116784:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0264f7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.116796:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b22680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+304:2388:x1870153002457344:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 265us (717us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.116804:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36364 00000100:00100000:0.0:1783520936.116813:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638587450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+304:2387:x1870153002457472:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 327us (677us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.116814:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516649880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993230848:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.116828:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36363 00010000:00010000:2.0:1783520936.116833:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0264fe lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.116882:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002457600 00000100:00100000:1.0:1783520936.116888:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002457600 00010000:00010000:1.0:1783520936.116915:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e9d0730c/0xd5401cbabc021c42 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f88 expref: 303 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.116945:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002457856 00000100:00100000:0.0:1783520936.116945:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002457728 00000100:00100000:1.0:1783520936.116950:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002457856 00000100:00100000:0.0:1783520936.116954:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002457728 00010000:00010000:1.0:1783520936.116966:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000cca6eef3/0xd5401cbabc021c57 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fa4 expref: 305 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.116985:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000bc40a014/0xd5401cbabc021c50 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f96 expref: 305 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.116999:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36365 00000100:00100000:1.0:1783520936.117025:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee5500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+305:2387:x1870153002457600:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.117026:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993231104:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.117041:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36366 00010000:00010000:1.0:1783520936.117046:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.117057:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e9d0730c/0xd5401cbabc021c42 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7d7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f88 expref: 305 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.117061:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+305:2390:x1870153002457856:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.117070:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026505 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.117086:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.117091:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36367 00010000:00010000:1.0:1783520936.117101:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e9d0730c/0xd5401cbabc021c42 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7d7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f88 expref: 305 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.117102:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000cca6eef3/0xd5401cbabc021c57 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7d9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fa4 expref: 305 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.117160:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000cca6eef3/0xd5401cbabc021c57 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7d9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fa4 expref: 305 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.117162:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d7:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.117180:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351664aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993231360:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.117180:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.117194:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cca6eef3/0xd5401cbabc021c57 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7d9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fa4 expref: 305 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.117195:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0264f0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.117207:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02650c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.117213:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503372a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+305:2387:x1870153002457728:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.117239:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.117243:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0264f0 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7d7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.117244:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.117261:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.117263:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000bc40a014/0xd5401cbabc021c50 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7d8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f96 expref: 304 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.117281:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0264fe lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.117291:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.117301:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000bc40a014/0xd5401cbabc021c50 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7d8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f96 expref: 304 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.117306:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0264fe lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.117330:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee5500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+304:2387:x1870153002457600:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 316us (765us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.117331:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.117342:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36365 00010000:00010000:3.0:1783520936.117435:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000bc40a014/0xd5401cbabc021c50 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7d8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f96 expref: 303 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.117437:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+303:2390:x1870153002457856:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 376us (675us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.117460:0:8410:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001cd9e5c0/0xd5401cbabc022923 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3013 expref: 303 pid: 9151 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.117487:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.117487:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36366 00010000:00010000:3.0:1783520936.117506:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.117515:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7d8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.117523:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0264f7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.117529:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b20e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993230208:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.117535:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002458112 00010000:00010000:3.0:1783520936.117541:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0264f7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.117544:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002458112 00010000:00010000:1.0:1783520936.117557:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0264e9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.117571:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0264e9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.117576:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f21ad4cb/0xd5401cbabc021c5e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fb2 expref: 301 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.117577:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.117589:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0264e9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.117604:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.117616:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503372a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+301:2387:x1870153002457728:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 466us (939us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.117627:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0264e9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.117632:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002457984 00000100:00100000:3.0:1783520936.117634:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36367 00000100:00100000:0.0:1783520936.117646:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002457984 00010000:00010000:1.0:1783520936.117647:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0264e9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.117651:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0264fe lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.117673:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005679b630/0xd5401cbabc021c65 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fc0 expref: 302 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.117691:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0264fe lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.117697:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.117709:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0264fe lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.117716:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36368 00010000:00010000:2.0:1783520936.117723:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0264fe lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.117728:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520936.117734:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351664aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+301:2389:x1870153002458112:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.117742:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0264e9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.117748:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.117751:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.117765:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f21ad4cb/0xd5401cbabc021c5e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7da:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fb2 expref: 301 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.117769:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.117780:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0264e9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.117791:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f21ad4cb/0xd5401cbabc021c5e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7da:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fb2 expref: 301 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.117797:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0264e9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.117819:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513bc0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+706:3628:x1870152993229696:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4078us (4413us total) trans 4294976274 rc 0/0 00010000:00010000:0.0:1783520936.117824:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f21ad4cb/0xd5401cbabc021c5e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7da:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fb2 expref: 301 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.117833:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0264e9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.117833:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44773 00010000:00010000:3.0:1783520936.117871:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0264e9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.117878:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7da:0x0].0x0 since=0s v=(0 0) 00000004:00080000:1.0:1783520936.117882:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513bc1500/2, rc 0, transno 4294976274 00010000:00010000:0.0:1783520936.117887:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7da:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.117897:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026505 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.117903:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520936.117906:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993229696:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.117916:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0264fe lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.117921:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032eaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+705:3627:x1870152993230720:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3157us (3246us total) trans 4294976275 rc 0/0 00000100:00100000:2.0:1783520936.117966:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44776 00010000:00010000:3.0:1783520936.117967:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.117969:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026505 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.117977:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0264fe lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.117979:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993231488:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.118009:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0264fe lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.118026:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0264fe lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.118028:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:0.0:1783520936.118038:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943508ee4a80/2, rc 0, transno 4294976275 00010000:00010000:3.0:1783520936.118039:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0264fe lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.118072:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993230720:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.118080:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993231488 00010000:00010000:3.0:1783520936.118082:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.118087:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993231488 00010000:00010000:3.0:1783520936.118091:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.118100:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36369 00010000:00010000:3.0:1783520936.118101:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.118112:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516649880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+300:2388:x1870153002457984:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.118119:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.118121:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.118128:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005679b630/0xd5401cbabc021c65 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7db:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fc0 expref: 300 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.118148:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0264f7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.118148:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44779 00010000:00010000:1.0:1783520936.118156:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005679b630/0xd5401cbabc021c65 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7db:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fc0 expref: 300 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.118180:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b21880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+705:3626:x1870152993231488:12345-0@lo:6:osp-syn-1-0.0 00000004:00080000:0.0:1783520936.118181:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513bc2d80/2, rc 0, transno 4294976273 00010000:00010000:3.0:1783520936.118187:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0264f7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.118195:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005679b630/0xd5401cbabc021c65 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7db:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fc0 expref: 300 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.118201:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993229184:0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.118206:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2012 count 1 00010000:00010000:3.0:1783520936.118227:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0264f7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.118235:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026513 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7dc:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.118238:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0264f7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.118239:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7db:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.118258:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7db:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.118271:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350626ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993231616:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.118277:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02650c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.118296:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001a6e7090/0xd5401cbabc021c6c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fce expref: 299 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.118310:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.118313:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02650c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.118336:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7dc:0x0].0x0 since=38s v=(0 0) 00000100:00100000:0.0:1783520936.118336:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.118353:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7dc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.118411:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.118420:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026513 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.118421:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1783520936.118438:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350626c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993231744:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.118445:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.118451:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0264f7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.118459:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.118467:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0264f7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.118476:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0264f7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.118501:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516649180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+706:3625:x1870152993230336:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4142us (4307us total) trans 4294976276 rc 0/0 00000100:00100000:1.0:1783520936.118502:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516649880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+299:2388:x1870153002457984:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 378us (898us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.118506:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0264f7 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.118522:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.118540:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44775 00000100:00100000:1.0:1783520936.118541:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36369 00010000:00010000:2.0:1783520936.118559:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc0264f7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.118592:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993231616 00000100:00100000:3.0:1783520936.118601:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993231616 00010000:00010000:2.0:1783520936.118603:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.118605:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02650c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.118610:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516649c00/2, rc 0, transno 4294976276 00010000:00010000:2.0:1783520936.118612:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.118622:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02650c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.118625:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.118636:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02650c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.118645:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02650c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.118648:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993231744 00000100:00100000:0.0:1783520936.118648:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516649c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993230336:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.118654:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993231744 00000100:00100000:2.0:1783520936.118662:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44780 00010000:00010000:1.0:1783520936.118667:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.118682:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.118690:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02650c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.118692:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44781 00000100:00100000:3.0:1783520936.118704:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350626d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+707:3627:x1870152993231744:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.118724:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184000 already unused, calling callback (ffffffffc0bf9e90) 00002000:00080000:3.0:1783520936.118732:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2014 count 1 00010000:00010000:2.0:1783520936.118733:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02650c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.118736:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:2.0:1783520936.118750:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02650c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.118752:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02651a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7de:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.118758:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350626df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993231872:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.118770:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b20a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+708:3626:x1870152993231232:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2804us (2909us total) trans 4294976277 rc 0/0 00000100:00100000:1.0:1783520936.118804:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44778 00010000:00010000:2.0:1783520936.118816:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02650c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.118817:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000049c28a07/0xd5401cbabc022b7d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f326d expref: 298 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.118842:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.118853:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x7de:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.118864:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7de:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.118870:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02651a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.118870:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02650c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.118896:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516649f80/2, rc 0, transno 4294976277 00000100:00100000:0.0:1783520936.118913:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516649f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993231232:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.118916:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000049c28a07/0xd5401cbabc022b7d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f326d expref: 298 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.118924:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.118936:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.118949:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.118949:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993231872 00010000:00010000:3.0:1783520936.118950:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000049c28a07/0xd5401cbabc022b7d lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f326d expref: 298 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.118960:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993231872 00000100:00100000:2.0:1783520936.118981:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635786c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+708:3627:x1870152993231616:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.118993:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44782 00002000:00080000:2.0:1783520936.119003:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2013 count 1 00000100:00100000:0.0:1783520936.119006:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350626ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993232128:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.119011:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350626ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+708:3626:x1870152993231872:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.119018:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503372300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993232000:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.119031:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026521 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7dd:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.119040:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2015 count 1 00010000:00010000:2.0:1783520936.119056:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000976689a8/0xd5401cbabc021c73 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fdc expref: 298 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.119058:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026528 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7df:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.119087:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6c00 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.119099:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.119103:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7dd:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.119105:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000fe739369/0xd5401cbabc021c88 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ff8 expref: 298 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.119114:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7dd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.119119:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7df:0x0].0x0 since=38s v=(0 0) 00000100:00100000:3.0:1783520936.119121:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993232128 00010000:00010000:2.0:1783520936.119124:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026521 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.119126:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7df:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.119127:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993232128 00010000:00010000:1.0:1783520936.119145:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026528 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.119162:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44783 00000100:00100000:3.0:1783520936.119171:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350626d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+709:3626:x1870152993232128:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.119179:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000976689a8/0xd5401cbabc021c73 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fdc expref: 298 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.119182:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000fe739369/0xd5401cbabc021c88 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ff8 expref: 298 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.119185:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2016 count 1 00010000:00010000:0.0:1783520936.119201:0:35282:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e9d0730c/0xd5401cbabc021c42 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1f88 expref: 298 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.119206:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02652f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7e0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.119234:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000976689a8/0xd5401cbabc021c73 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fdc expref: 298 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.119273:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000071aa3eac/0xd5401cbabc021c8f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2006 expref: 298 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.119300:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000fe739369/0xd5401cbabc021c88 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ff8 expref: 298 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.119310:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516648380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993232256:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.119319:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032e9500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993230464:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.119324:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e0:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.119340:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.119347:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02652f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.119349:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b20000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993232384:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.119391:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0264f0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.119406:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0264f0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.119416:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0264f0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.119427:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0264f0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.119439:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0264f0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.119465:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.119476:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.119484:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0264f0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.119486:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470e40 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.119487:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435131fc300 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.119513:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.119519:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0264f0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.119530:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0264f0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.119551:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0264f0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.119556:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001a6e7090/0xd5401cbabc021c6c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fce expref: 297 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.119576:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0264f0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.119604:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b23100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+710:3626:x1870152993230080:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5328us (5509us total) trans 4294976278 rc 0/0 00010000:00010000:2.0:1783520936.119626:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001a6e7090/0xd5401cbabc021c6c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fce expref: 297 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.119627:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44774 00010000:00010000:3.0:1783520936.119634:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.119642:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.119648:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.119650:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351664ad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993232512:192.168.204.33@tcp:104: 00000004:00080000:0.0:1783520936.119675:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435032e9880/2, rc 0, transno 4294976278 00010000:00010000:3.0:1783520936.119684:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000071aa3eac/0xd5401cbabc021c8f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2006 expref: 297 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.119697:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435131fc180 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.119697:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993230080:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.119701:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000071aa3eac/0xd5401cbabc021c8f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2006 expref: 297 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.119722:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503371880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993232640:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.119738:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.119754:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026505 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.119765:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351664aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+297:2389:x1870153002458112:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 2036us (2298us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.119768:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026505 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.119780:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36368 00010000:00010000:2.0:1783520936.119782:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026505 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.119794:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026505 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.119796:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6540 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.119828:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0264e2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.119828:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.119836:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993232768:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.119855:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0264e2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.119870:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0264e2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.119880:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0264e2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.119894:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.119907:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.119917:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026505 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.119928:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.119933:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026505 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.119945:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351664bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+710:3628:x1870152993230976:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4660us (4762us total) trans 4294976279 rc 0/0 00010000:00010000:3.0:1783520936.119950:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026505 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.119956:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350626e300/2, rc 0, transno 4294976279 00000100:00100000:2.0:1783520936.119960:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44777 00000100:00100000:0.0:1783520936.119977:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350626e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993230976:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.119986:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993232768 00000100:00100000:2.0:1783520936.119991:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993232768 00000100:00100000:2.0:1783520936.120029:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44784 00000100:00100000:0.0:1783520936.120031:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350626c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993232896:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.120045:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350626dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+710:3627:x1870152993232768:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.120063:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2017 count 1 00010000:00010000:2.0:1783520936.120080:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026536 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7e1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.120091:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.120095:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026505 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.120096:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005f9597e0/0xd5401cbabc021c96 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2014 expref: 296 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.120130:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026505 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.120133:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e1:0x0].0x0 since=38s v=(0 0) 00000100:00100000:0.0:1783520936.120134:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993232896 00000100:00100000:0.0:1783520936.120145:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993232896 00010000:00010000:2.0:1783520936.120146:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.120167:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026536 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.120192:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.120197:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44785 00010000:00010000:3.0:1783520936.120199:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.120205:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.120206:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005f9597e0/0xd5401cbabc021c96 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2014 expref: 296 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.120214:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350626e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+711:3628:x1870152993232896:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.120230:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005f9597e0/0xd5401cbabc021c96 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2014 expref: 296 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.120231:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00002000:00080000:0.0:1783520936.120232:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2018 count 1 00000100:00100000:2.0:1783520936.120250:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516648000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993233024:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.120253:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02653d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7e2:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.120266:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.120268:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000fac66d00/0xd5401cbabc021c9d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2022 expref: 296 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.120272:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0264e2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.120283:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e2:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.120290:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.120291:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.120296:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02653d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.120299:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0264e2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.120321:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0264e2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.120331:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000fac66d00/0xd5401cbabc021c9d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2022 expref: 296 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.120336:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0264e2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.120408:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000fac66d00/0xd5401cbabc021c9d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2022 expref: 296 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.120421:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513bc0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+712:3627:x1870152993229440:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6838us (7292us total) trans 4294976280 rc 0/0 00010000:00010000:2.0:1783520936.120423:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc0264e2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.120430:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513bc2300/2, rc 0, transno 4294976280 00000100:00100000:0.0:1783520936.120434:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350626ea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993233152:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.120454:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44772 00000100:00100000:1.0:1783520936.120465:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993229440:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.120522:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:2.0:1783520936.120530:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.120530:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350626f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993233280:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.120534:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.120539:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.120557:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.120596:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993233280 00000100:00100000:1.0:1783520936.120605:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993233280 00000100:00100000:2.0:1783520936.120635:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e300 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.120660:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895600 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.120663:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44786 00000100:00100000:1.0:1783520936.120675:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032e9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+712:3626:x1870152993233280:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.120692:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2019 count 1 00010000:00010000:1.0:1783520936.120709:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026544 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7e3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.120725:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007295bd55/0xd5401cbabc021cab lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2030 expref: 296 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.120734:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e3:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.120738:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.120743:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026544 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.120776:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007295bd55/0xd5401cbabc021cab lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2030 expref: 296 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.120788:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007295bd55/0xd5401cbabc021cab lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2030 expref: 296 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.120795:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503372300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993232000:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.120824:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b23480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993233408:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.120834:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02651a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.120861:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.120942:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b20000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993232384:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.120959:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026528 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.121048:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516648380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993232256:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.121078:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026521 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.121083:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351664ad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993232512:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.121124:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026513 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.121128:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503371880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993232640:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.121159:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02652f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.121909:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.121936:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.121962:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002458240 00000100:00100000:1.0:1783520936.121964:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.121971:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002458240 00000100:00100000:3.0:1783520936.121991:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002458496 00000100:00100000:3.0:1783520936.121999:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002458496 00010000:00010000:2.0:1783520936.122006:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000049c28a07/0xd5401cbabc022b7d lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f326d expref: 298 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.122023:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.122025:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002458752 00010000:00010000:3.0:1783520936.122029:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000976689a8/0xd5401cbabc021c73 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fdc expref: 298 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.122034:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002458752 00000100:00100000:2.0:1783520936.122045:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002458368 00000100:00100000:2.0:1783520936.122053:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002458368 00010000:00010000:0.0:1783520936.122071:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000071aa3eac/0xd5401cbabc021c8f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2006 expref: 300 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.122081:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36370 00010000:00010000:2.0:1783520936.122082:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000fe739369/0xd5401cbabc021c88 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ff8 expref: 300 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.122086:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.122106:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b20000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+300:2387:x1870153002458240:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.122136:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.122139:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002458624 00000100:00100000:0.0:1783520936.122146:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002458624 00010000:00010000:3.0:1783520936.122149:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000049c28a07/0xd5401cbabc022b7d lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7de:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f326d expref: 300 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.122164:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36371 00010000:00010000:0.0:1783520936.122170:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001a6e7090/0xd5401cbabc021c6c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fce expref: 301 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.122174:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000049c28a07/0xd5401cbabc022b7d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7de:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f326d expref: 301 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.122187:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638586c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+301:2387:x1870153002458496:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.122199:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.122225:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.122227:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002458880 00000100:00100000:0.0:1783520936.122232:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002458880 00010000:00010000:3.0:1783520936.122235:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000049c28a07/0xd5401cbabc022b7d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7de:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f326d expref: 301 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.122236:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000976689a8/0xd5401cbabc021c73 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7dd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fdc expref: 301 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.122255:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005f9597e0/0xd5401cbabc021c96 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2014 expref: 302 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.122269:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.122278:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000976689a8/0xd5401cbabc021c73 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7dd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fdc expref: 302 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.122307:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002459008 00010000:00010000:2.0:1783520936.122308:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000976689a8/0xd5401cbabc021c73 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7dd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fdc expref: 302 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.122310:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002459008 00000100:00100000:1.0:1783520936.122322:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36372 00010000:00010000:0.0:1783520936.122328:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000fac66d00/0xd5401cbabc021c9d lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2022 expref: 302 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.122330:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7dd:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.122337:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b21f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+302:2387:x1870153002458752:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.122348:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7dd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.122355:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.122396:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026521 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.122404:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000071aa3eac/0xd5401cbabc021c8f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7e0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2006 expref: 302 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.122415:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36373 00010000:00010000:2.0:1783520936.122441:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026521 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.122442:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b23b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+302:2387:x1870153002458368:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.122459:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.122471:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000071aa3eac/0xd5401cbabc021c8f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7e0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2006 expref: 302 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.122471:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000fe739369/0xd5401cbabc021c88 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7df:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ff8 expref: 302 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.122486:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.122505:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7de:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.122521:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7de:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.122522:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000fe739369/0xd5401cbabc021c88 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7df:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ff8 expref: 301 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.122538:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000071aa3eac/0xd5401cbabc021c8f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7e0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2006 expref: 301 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.122541:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02651a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.122544:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638586c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+301:2387:x1870153002458496:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 363us (599us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.122573:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000fe739369/0xd5401cbabc021c88 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7df:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1ff8 expref: 301 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.122592:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e0:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.122594:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36371 00010000:00010000:1.0:1783520936.122612:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.122613:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02651a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.122627:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02652f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.122630:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7df:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.122640:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36374 00010000:00010000:3.0:1783520936.122645:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.122657:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7df:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.122658:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02652f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.122672:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b20380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+298:2387:x1870153002458624:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.122673:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026528 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.122692:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.122705:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b20000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+298:2387:x1870153002458240:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 599us (786us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.122710:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026528 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.122711:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001a6e7090/0xd5401cbabc021c6c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7dc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fce expref: 298 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.122732:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.122749:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36370 00010000:00010000:0.0:1783520936.122766:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.122767:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b21f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+298:2387:x1870153002458752:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 429us (796us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.122768:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001a6e7090/0xd5401cbabc021c6c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7dc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fce expref: 298 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.122804:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b23b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+297:2387:x1870153002458368:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 362us (778us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.122814:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36372 00010000:00010000:2.0:1783520936.122816:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001a6e7090/0xd5401cbabc021c6c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7dc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f1fce expref: 297 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.122830:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36373 00000100:00100000:3.0:1783520936.122843:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36375 00010000:00010000:2.0:1783520936.122846:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7dc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.122854:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7dc:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.122860:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36376 00000100:00100000:3.0:1783520936.122862:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b22d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+294:2387:x1870153002458880:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.122863:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026513 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.122872:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.122873:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b20700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+294:2390:x1870153002459008:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.122885:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005f9597e0/0xd5401cbabc021c96 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7e1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2014 expref: 294 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.122893:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.122894:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026513 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.122908:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000fac66d00/0xd5401cbabc021c9d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7e2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2022 expref: 294 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.122908:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350626ea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993233152:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.122946:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02653d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7e2:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.122954:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005f9597e0/0xd5401cbabc021c96 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7e1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2014 expref: 294 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.122982:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.123013:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b20380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+294:2387:x1870153002458624:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 347us (942us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.123022:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.123023:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000fac66d00/0xd5401cbabc021c9d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7e2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2022 expref: 294 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.123024:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026513 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.123034:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e1:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.123035:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36374 00010000:00010000:3.0:1783520936.123057:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026536 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.123080:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026513 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.123087:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026536 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7e1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.123094:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000fac66d00/0xd5401cbabc021c9d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7e2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2022 expref: 293 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.123106:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026513 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.123133:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026513 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.123134:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.123135:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.123151:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.123171:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02653d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.123178:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b22d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+292:2387:x1870153002458880:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 321us (990us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.123182:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02653d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.123185:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520936.123198:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36375 00000100:00100000:2.0:1783520936.123220:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.123221:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.123254:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.123263:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026513 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.123271:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b20700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+291:2390:x1870153002459008:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 395us (1007us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.123291:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36376 00010000:00010000:2.0:1783520936.123302:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.123313:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026513 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.123328:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002459136 00010000:00010000:2.0:1783520936.123329:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026513 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.123334:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002459136 00000100:00100000:0.0:1783520936.123337:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b21880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+713:3626:x1870152993231488:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5169us (5305us total) trans 4294976281 rc 0/0 00000100:00100000:0.0:1783520936.123350:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44779 00010000:00010000:1.0:1783520936.123401:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007295bd55/0xd5401cbabc021cab lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2030 expref: 291 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.123402:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026513 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.123432:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513b20e00/2, rc 0, transno 4294976281 00010000:00010000:2.0:1783520936.123434:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026513 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.123452:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993231488:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.123470:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36377 00010000:00010000:2.0:1783520936.123484:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.123485:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516648380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+291:2387:x1870153002459136:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.123503:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.123510:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.123512:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.123523:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007295bd55/0xd5401cbabc021cab lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7e3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2030 expref: 291 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.123530:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026521 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.123538:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350626ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993233536:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.123546:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007295bd55/0xd5401cbabc021cab lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7e3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2030 expref: 291 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.123555:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026521 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.123579:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.123587:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026521 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.123594:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.123610:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.123620:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026521 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.123624:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026544 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.123632:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993233536 00010000:00010000:0.0:1783520936.123633:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02653d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.123650:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993233536 00010000:00010000:2.0:1783520936.123685:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.123687:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02653d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.123693:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026544 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7e3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.123709:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.123711:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02653d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.123721:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026521 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.123723:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.123727:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02653d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.123764:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516648380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+291:2387:x1870153002459136:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 280us (548us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.123785:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.123800:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026521 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.123800:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36377 00010000:00010000:0.0:1783520936.123808:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.123816:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635786c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+714:3627:x1870152993231616:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5135us (5481us total) trans 4294976282 rc 0/0 00010000:00010000:1.0:1783520936.123843:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02652f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.123850:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026521 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.123853:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44780 00010000:00010000:1.0:1783520936.123880:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02652f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.123894:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026521 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.123896:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02652f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.123920:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350626e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+714:3628:x1870152993232896:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3703us (3823us total) trans 4294976283 rc 0/0 00010000:00010000:3.0:1783520936.123921:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026521 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.123924:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02652f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.123963:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44785 00010000:00010000:1.0:1783520936.124005:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520936.124021:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520936.124026:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02653d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.124029:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350626c380/2, rc 0, transno 4294976283 00010000:00010000:3.0:1783520936.124046:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea800 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520936.124046:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350626c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993232896:0@lo:6:osp-syn-1-0.0 00000004:00080000:2.0:1783520936.124055:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350626ca80/2, rc 0, transno 4294976282 00010000:00010000:3.0:1783520936.124057:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02653d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.124067:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350626ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993231616:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.124091:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02653d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.124099:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993233664:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.124122:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993233792:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.124123:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02653d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.124143:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc02653d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.124174:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.124187:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.124192:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02651a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.124194:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02652f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.124200:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350626d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+714:3626:x1870152993232128:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5026us (5113us total) trans 4294976284 rc 0/0 00010000:00010000:2.0:1783520936.124228:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.124240:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44783 00010000:00010000:3.0:1783520936.124243:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.124244:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02651a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.124249:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993233664 00010000:00010000:3.0:1783520936.124254:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02652f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.124257:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993233664 00010000:00010000:0.0:1783520936.124281:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02651a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.124286:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02652f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.124289:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02651a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.124300:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350626ce00/2, rc 0, transno 4294976284 00000100:00100000:2.0:1783520936.124312:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44787 00000100:00100000:1.0:1783520936.124331:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350626ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993232128:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.124338:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+714:3626:x1870152993233536:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.124351:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02652f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.124383:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.124395:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc02652f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.124413:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2020 count 1 00010000:00010000:3.0:1783520936.124432:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.124438:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.124441:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02654b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7e4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.124444:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.124458:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to regular blp list, 0 blwis in pool 00010000:00010000:3.0:1783520936.124460:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02651a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.124472:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008bc2a35a/0xd5401cbabc021cb2 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f203e expref: 290 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.124476:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.124487:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02651a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.124509:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e4:0x0].0x0 since=38s v=(0 0) 00000100:00100000:0.0:1783520936.124514:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993233920:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.124517:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02651a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.124520:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e4:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.124536:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993233792 00010000:00010000:2.0:1783520936.124541:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02654b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.124543:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993233792 00000100:00100000:0.0:1783520936.124554:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.124559:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02651a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.124573:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc02651a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.124574:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.124585:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993233920 00000100:00100000:1.0:1783520936.124591:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993233920 00010000:00010000:3.0:1783520936.124604:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.124611:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.124615:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008bc2a35a/0xd5401cbabc021cb2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f203e expref: 290 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.124617:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.124631:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44788 00010000:00010000:3.0:1783520936.124641:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.124650:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008bc2a35a/0xd5401cbabc021cb2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f203e expref: 290 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.124652:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351664a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+717:3627:x1870152993233664:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.124673:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44789 00002000:00080000:1.0:1783520936.124686:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2021 count 1 00000100:00100000:2.0:1783520936.124691:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516649500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993234048:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.124699:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350626d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+717:3627:x1870152993231744:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5985us (6180us total) trans 4294976285 rc 0/0 00000100:00100000:3.0:1783520936.124700:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+717:3628:x1870152993233792:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.124718:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026552 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7e5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.124732:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2022 count 1 00000100:00100000:0.0:1783520936.124736:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44781 00010000:00010000:1.0:1783520936.124759:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000114508e0/0xd5401cbabc021cb9 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f204c expref: 290 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.124763:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026559 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7e6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.124773:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.124780:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44790 00010000:00010000:1.0:1783520936.124785:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e5:0x0].0x0 since=38s v=(0 0) 00000100:00100000:0.0:1783520936.124801:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+716:3625:x1870152993233920:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.124802:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.124818:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006cbd4d94/0xd5401cbabc022a6c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f315c expref: 290 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.124820:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350626c700/2, rc 0, transno 4294976285 00010000:00010000:1.0:1783520936.124825:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026552 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.124833:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2023 count 1 00000100:00100000:2.0:1783520936.124849:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350626c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993231744:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.124863:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026560 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7e7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.124866:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x7e6:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.124887:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.124899:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026559 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.124901:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000114508e0/0xd5401cbabc021cb9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f204c expref: 290 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.124905:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008d8af792/0xd5401cbabc021cce lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2068 expref: 290 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.124954:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e7:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.124962:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006cbd4d94/0xd5401cbabc022a6c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f315c expref: 290 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.124962:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000114508e0/0xd5401cbabc021cb9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f204c expref: 290 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.124963:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.124990:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026560 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.125010:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006cbd4d94/0xd5401cbabc022a6c lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f315c expref: 290 pid: 20415 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.125019:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b20a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993234176:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.125028:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1783520936.125037:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503371f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993234432:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.125046:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516649180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993234304:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.125074:0:9151:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005f9597e0/0xd5401cbabc021c96 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2014 expref: 290 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.125076:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026528 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.125092:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.125115:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026528 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.125129:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516648000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993233024:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.125131:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026528 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.125143:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993234304 00010000:00010000:1.0:1783520936.125150:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026528 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.125151:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993234304 00000100:00100000:3.0:1783520936.125158:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6f00 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.125160:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026536 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.125177:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026536 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.125182:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44791 00010000:00010000:1.0:1783520936.125185:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.125193:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+717:3625:x1870152993234304:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.125200:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026536 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.125207:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2024 count 1 00010000:00010000:2.0:1783520936.125211:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026536 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.125219:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026567 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7e8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.125221:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026536 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.125245:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003cf00aee/0xd5401cbabc021cd5 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2076 expref: 289 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.125252:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.125255:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e8:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.125259:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026528 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.125260:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.125269:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026567 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.125273:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.125280:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026528 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.125291:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026528 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.125294:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003cf00aee/0xd5401cbabc021cd5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2076 expref: 289 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.125311:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350626ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+718:3626:x1870152993231872:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6303us (6483us total) trans 4294976286 rc 0/0 00010000:00010000:0.0:1783520936.125321:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003cf00aee/0xd5401cbabc021cd5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2076 expref: 289 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.125322:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026528 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.125335:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44782 00010000:00010000:2.0:1783520936.125355:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026528 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.125390:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993234560:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.125430:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.125436:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.125439:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520936.125448:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350626df80/2, rc 0, transno 4294976286 00000100:00100000:0.0:1783520936.125474:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895600 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.125476:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350626df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993231872:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.125477:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.125532:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008d8af792/0xd5401cbabc021cce lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2068 expref: 289 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.125550:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008d8af792/0xd5401cbabc021cce lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2068 expref: 289 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.125561:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993234688:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.125569:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.125578:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026536 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.125590:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520936.125594:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895840 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.125595:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026536 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.125608:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026536 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.125623:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026536 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.125631:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026536 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.125659:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.125663:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.125665:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.125669:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470e40 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.125684:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1783520936.125698:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516648000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993234816:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.125720:0:8410:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007295bd55/0xd5401cbabc021cab lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2030 expref: 289 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.125739:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b23480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993233408:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.125748:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.125765:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026544 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.125777:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026544 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.125785:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026544 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.125795:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026544 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.125799:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993234816 00010000:00010000:1.0:1783520936.125809:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026544 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.125810:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993234816 00010000:00010000:1.0:1783520936.125837:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.125849:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44792 00000100:00100000:2.0:1783520936.125862:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351664bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+718:3628:x1870152993234816:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.125869:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.125877:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026544 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.125879:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2025 count 1 00010000:00010000:0.0:1783520936.125891:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.125898:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026544 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.125900:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02656e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7e9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.125919:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026544 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.125935:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026544 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.125943:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026544 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.125946:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032e9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+719:3626:x1870152993233280:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5271us (5386us total) trans 4294976287 rc 0/0 00000100:00100000:1.0:1783520936.125972:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44786 00010000:00010000:0.0:1783520936.125985:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.125990:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.125997:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.126008:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000006b7649a/0xd5401cbabc021cdc lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2084 expref: 288 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.126016:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e9:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.126021:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e9:0x0].0x0 since=0s v=(0 0) 00000004:00080000:1.0:1783520936.126024:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350626f800/2, rc 0, transno 4294976287 00010000:00010000:2.0:1783520936.126026:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02656e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.126048:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350626f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993233280:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.126082:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503371f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993234432:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.126101:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:0.0:1783520936.126108:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026559 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.126113:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516648e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993234944:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.126158:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.126177:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.126192:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993234944 00000100:00100000:0.0:1783520936.126200:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993234944 00000100:00100000:2.0:1783520936.126232:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516649500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993234048:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.126260:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44793 00010000:00010000:2.0:1783520936.126266:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02654b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.126277:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351664b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+719:3627:x1870152993234944:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.126291:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2026 count 1 00010000:00010000:2.0:1783520936.126303:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000006b7649a/0xd5401cbabc021cdc lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2084 expref: 288 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.126304:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026575 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7ea:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.126327:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000006b7649a/0xd5401cbabc021cdc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2084 expref: 288 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.126331:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000000ff0eacb/0xd5401cbabc021ce3 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2092 expref: 288 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.126353:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ea:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.126414:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ea:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.126419:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026575 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.126420:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002459264 00000100:00100000:2.0:1783520936.126426:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516649500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993235072:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.126427:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002459264 00000100:00100000:3.0:1783520936.126448:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.126466:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008bc2a35a/0xd5401cbabc021cb2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f203e expref: 289 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.126471:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000000ff0eacb/0xd5401cbabc021ce3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2092 expref: 289 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.126517:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000000ff0eacb/0xd5401cbabc021ce3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2092 expref: 289 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.126524:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.126532:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002459392 00000100:00100000:0.0:1783520936.126538:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943505877480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993235200:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.126539:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002459392 00010000:00010000:1.0:1783520936.126567:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006cbd4d94/0xd5401cbabc022a6c lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f315c expref: 290 pid: 20415 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.126575:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36378 00000100:00100000:0.0:1783520936.126590:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895c00 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.126602:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638585850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+290:2388:x1870153002459264:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.126619:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.126627:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36379 00010000:00010000:2.0:1783520936.126630:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008bc2a35a/0xd5401cbabc021cb2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7e4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f203e expref: 290 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.126650:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993234560:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.126652:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503372a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+290:2387:x1870153002459392:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.126653:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008bc2a35a/0xd5401cbabc021cb2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7e4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f203e expref: 290 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.126680:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.126687:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008bc2a35a/0xd5401cbabc021cb2 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7e4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f203e expref: 290 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.126690:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006cbd4d94/0xd5401cbabc022a6c lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7e6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f315c expref: 290 pid: 20415 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.126697:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026567 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.126725:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.126738:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.126744:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02654b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.126748:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006cbd4d94/0xd5401cbabc022a6c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7e6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f315c expref: 289 pid: 20415 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.126757:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02654b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.126799:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.126800:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006cbd4d94/0xd5401cbabc022a6c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7e6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f315c expref: 289 pid: 20415 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.126820:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.126823:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02654b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.126825:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e6:0x0].0x0 since=0s v=(1 0) 00000100:00100000:2.0:1783520936.126837:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638585850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+288:2388:x1870153002459264:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 240us (657us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.126839:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026559 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.126859:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36378 00010000:00010000:1.0:1783520936.126871:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026559 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.126904:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.126945:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026559 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.126957:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026559 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.126973:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026559 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.126981:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026559 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.126992:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503372a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+287:2387:x1870153002459392:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 353us (546us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.127003:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.127004:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36379 00010000:00010000:3.0:1783520936.127023:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02654b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.127037:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02654b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.127049:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02654b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.127059:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350626dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+721:3627:x1870152993232768:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7017us (7169us total) trans 4294976288 rc 0/0 00000100:00100000:2.0:1783520936.127073:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44784 00010000:00010000:3.0:1783520936.127077:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.127086:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026559 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.127097:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+720:3628:x1870152993233792:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2407us (2738us total) trans 4294976289 rc 0/0 00000100:00100000:0.0:1783520936.127113:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44789 00000004:00080000:2.0:1783520936.127116:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513b22a00/2, rc 0, transno 4294976288 00010000:00010000:3.0:1783520936.127131:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4a00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520936.127134:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993232768:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.127139:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026559 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.127151:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026559 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.127168:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026559 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.127177:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026559 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.127186:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516648380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993235328:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.127214:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.127245:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993235328 00000100:00100000:1.0:1783520936.127252:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993235328 00010000:00010000:3.0:1783520936.127277:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.127279:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.127281:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.127293:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44794 00000100:00100000:1.0:1783520936.127304:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505876d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+720:3625:x1870152993235328:12345-0@lo:6:osp-syn-1-0.0 00000004:00080000:3.0:1783520936.127313:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e9a680/2, rc 0, transno 4294976289 00002000:00080000:1.0:1783520936.127323:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2027 count 1 00000100:00100000:3.0:1783520936.127329:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993233792:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.127342:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02657c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7eb:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.127405:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.127406:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e19945ab/0xd5401cbabc021cea lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f20a0 expref: 286 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.127417:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7eb:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.127420:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7eb:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.127427:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351664b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993235456:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.127428:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02657c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.127430:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.127437:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02654b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.127479:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185400 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520936.127484:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.127485:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02654b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.127509:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02654b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.127528:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02654b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.127535:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02654b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.127546:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993235456 00000100:00100000:1.0:1783520936.127554:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993235456 00010000:00010000:2.0:1783520936.127555:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.127559:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.127563:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520936.127563:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350626ea00/2, rc 0, transno 4294976290 00000100:00100000:0.0:1783520936.127579:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350626ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993233536:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.127610:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44795 00000100:00100000:1.0:1783520936.127626:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505875c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+722:3626:x1870152993235456:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.127634:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00002000:00080000:1.0:1783520936.127644:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2028 count 1 00000100:00100000:3.0:1783520936.127646:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+722:3626:x1870152993233536:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3309us (4066us total) trans 4294976290 rc 0/0 00000100:00100000:2.0:1783520936.127652:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516649880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993235584:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.127664:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026583 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7ec:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.127674:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44787 00010000:00010000:1.0:1783520936.127684:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000003a0ec53/0xd5401cbabc021cf1 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f20ae expref: 286 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.127698:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ec:0x0].0x0 since=38s v=(0 0) 00000100:00100000:2.0:1783520936.127705:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.127707:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ec:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.127715:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026583 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.127751:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000003a0ec53/0xd5401cbabc021cf1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20ae expref: 286 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.127765:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000003a0ec53/0xd5401cbabc021cf1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20ae expref: 286 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.127780:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b23480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993235712:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.127794:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993234688:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.127822:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026560 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.127849:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470780 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.127855:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993235584 00000100:00100000:0.0:1783520936.127860:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993235584 00000100:00100000:0.0:1783520936.127890:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44796 00010000:00010000:1.0:1783520936.127896:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e19945ab/0xd5401cbabc021cea lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20a0 expref: 286 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.127908:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351664aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+722:3628:x1870152993235584:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.127910:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b20a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993234176:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.127943:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e19945ab/0xd5401cbabc021cea lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20a0 expref: 286 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.127947:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026552 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.127949:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2029 count 1 00000100:00100000:1.0:1783520936.127972:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b23100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993235840:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.127979:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02658a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7ed:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.127987:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.128009:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008b1d41a4/0xd5401cbabc021cff lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f20bc expref: 286 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.128015:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470b40 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.128018:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ed:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.128025:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ed:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.128031:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02658a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.128103:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.128460:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002459520 00010000:00010000:1.0:1783520936.128466:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008b1d41a4/0xd5401cbabc021cff lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20bc expref: 286 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.128468:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002459520 00010000:00010000:1.0:1783520936.128485:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008b1d41a4/0xd5401cbabc021cff lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20bc expref: 287 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.128511:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003cf00aee/0xd5401cbabc021cd5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2076 expref: 287 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.128518:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.128520:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b21880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993235968:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.128578:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002459648 00000100:00100000:2.0:1783520936.128581:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002459776 00000100:00100000:0.0:1783520936.128584:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002459648 00000100:00100000:2.0:1783520936.128590:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002459776 00000100:00100000:1.0:1783520936.128601:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470480 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.128608:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000114508e0/0xd5401cbabc021cb9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f204c expref: 289 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.128618:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008d8af792/0xd5401cbabc021cce lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2068 expref: 289 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.128655:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36380 00000100:00100000:2.0:1783520936.128668:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36381 00000100:00100000:0.0:1783520936.128670:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+289:2388:x1870153002459520:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.128682:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503370e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+289:2389:x1870153002459648:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.128682:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.128690:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003cf00aee/0xd5401cbabc021cd5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7e8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2076 expref: 289 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.128698:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.128706:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003cf00aee/0xd5401cbabc021cd5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7e8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2076 expref: 289 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.128710:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000114508e0/0xd5401cbabc021cb9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7e5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f204c expref: 289 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.128714:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.128757:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000114508e0/0xd5401cbabc021cb9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7e5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f204c expref: 289 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.128760:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003cf00aee/0xd5401cbabc021cd5 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7e8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2076 expref: 289 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.128771:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002459904 00000100:00100000:3.0:1783520936.128786:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002459904 00000100:00100000:1.0:1783520936.128790:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.128792:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.128796:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000114508e0/0xd5401cbabc021cb9 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7e5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f204c expref: 289 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.128799:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.128815:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026567 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.128817:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000006b7649a/0xd5401cbabc021cdc lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2084 expref: 289 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.128822:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.128843:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e5:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.128845:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002460032 00000100:00100000:1.0:1783520936.128850:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002460032 00010000:00010000:2.0:1783520936.128866:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026552 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.128868:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026567 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.128879:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000000ff0eacb/0xd5401cbabc021ce3 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2092 expref: 289 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.128886:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36382 00000100:00100000:3.0:1783520936.128909:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+289:2390:x1870153002459776:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.128909:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026552 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.128939:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.128950:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36383 00010000:00010000:2.0:1783520936.128959:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.128964:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b20000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+289:2388:x1870153002459904:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.128977:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.128986:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+289:2388:x1870153002459520:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 318us (1001us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.128992:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000006b7649a/0xd5401cbabc021cdc lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7e9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2084 expref: 289 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.129004:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.129009:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36380 00000100:00100000:2.0:1783520936.129011:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503370e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+289:2389:x1870153002459648:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 324us (898us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.129022:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008d8af792/0xd5401cbabc021cce lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7e7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2068 expref: 289 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.129032:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000006b7649a/0xd5401cbabc021cdc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7e9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2084 expref: 288 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.129053:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36384 00000100:00100000:2.0:1783520936.129071:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36381 00000100:00100000:0.0:1783520936.129076:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b21180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+288:2389:x1870153002460032:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.129077:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.129090:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.129101:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02656e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.129106:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.129111:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026552 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.129133:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000000ff0eacb/0xd5401cbabc021ce3 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7ea:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2092 expref: 287 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.129135:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02656e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7e9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.129136:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008d8af792/0xd5401cbabc021cce lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7e7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2068 expref: 287 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.129176:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026552 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.129207:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.129221:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026552 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.129226:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000000ff0eacb/0xd5401cbabc021ce3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ea:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2092 expref: 287 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.129237:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026552 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.129243:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b20000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+287:2388:x1870153002459904:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 279us (531us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.129266:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ea:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.129270:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008d8af792/0xd5401cbabc021cce lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7e7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2068 expref: 287 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.129280:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ea:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.129289:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36383 00010000:00010000:2.0:1783520936.129291:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.129302:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026575 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ea:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.129313:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.129314:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.129314:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026575 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7ea:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.129328:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7e7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.129330:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026552 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.129343:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026560 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.129348:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b23480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993235712:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.129403:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.129422:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.129426:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026552 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.129432:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026583 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.129437:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026552 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.129444:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b21180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+285:2389:x1870153002460032:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 369us (655us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.129487:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36384 00010000:00010000:3.0:1783520936.129509:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026560 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.129512:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026552 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.129538:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026552 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.129563:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.129569:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.129574:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.129575:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.129581:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.129599:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+284:2390:x1870153002459776:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 693us (1083us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.129609:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b23100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993235840:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.129614:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36382 00010000:00010000:1.0:1783520936.129644:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02657c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.129647:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002460160 00000100:00100000:3.0:1783520936.129652:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002460160 00000100:00100000:0.0:1783520936.129667:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.129679:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000003a0ec53/0xd5401cbabc021cf1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20ae expref: 284 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.129722:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002460288 00000100:00100000:3.0:1783520936.129727:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002460288 00000100:00100000:1.0:1783520936.129732:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36385 00000100:00100000:1.0:1783520936.129744:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638584050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+285:2388:x1870153002460160:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.129750:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e19945ab/0xd5401cbabc021cea lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20a0 expref: 285 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.129752:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.129763:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000003a0ec53/0xd5401cbabc021cf1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ec:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20ae expref: 285 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.129768:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351664a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+723:3627:x1870152993233664:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5117us (5596us total) trans 4294976291 rc 0/0 00000100:00100000:3.0:1783520936.129791:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36386 00010000:00010000:1.0:1783520936.129794:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000003a0ec53/0xd5401cbabc021cf1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7ec:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20ae expref: 285 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.129796:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44788 00000100:00100000:3.0:1783520936.129808:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505874e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+285:2388:x1870153002460288:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.129822:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.129829:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000003a0ec53/0xd5401cbabc021cf1 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7ec:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20ae expref: 285 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.129830:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e19945ab/0xd5401cbabc021cea lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7eb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20a0 expref: 285 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.129855:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000004:00080000:2.0:1783520936.129858:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e9b100/2, rc 0, transno 4294976291 00010000:00010000:1.0:1783520936.129860:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ec:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.129862:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e19945ab/0xd5401cbabc021cea lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7eb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20a0 expref: 284 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.129868:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ec:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.129874:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993233664:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.129885:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026583 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.129893:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e19945ab/0xd5401cbabc021cea lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7eb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20a0 expref: 284 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.129900:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026583 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.129912:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002460416 00000100:00100000:0.0:1783520936.129916:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002460416 00010000:00010000:1.0:1783520936.129929:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.129933:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351664ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993236096:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.129938:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008b1d41a4/0xd5401cbabc021cff lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20bc expref: 285 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.129966:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36387 00000100:00100000:1.0:1783520936.129979:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638584050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+285:2388:x1870153002460160:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 232us (407us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.129983:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505877800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+284:2387:x1870153002460416:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.129986:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7eb:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.129989:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.130006:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7eb:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.130015:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36385 00010000:00010000:3.0:1783520936.130016:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02657c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.130016:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.130035:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008b1d41a4/0xd5401cbabc021cff lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7ed:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20bc expref: 284 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.130038:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02657c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.130071:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993236096 00000100:00100000:2.0:1783520936.130079:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993236096 00010000:00010000:0.0:1783520936.130081:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008b1d41a4/0xd5401cbabc021cff lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ed:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20bc expref: 283 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.130090:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.130102:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ed:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.130108:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ed:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.130113:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02658a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ed:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.130125:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02658a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7ed:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.130127:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505874e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+283:2388:x1870153002460288:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 319us (466us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.130127:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44797 00000100:00100000:2.0:1783520936.130156:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032eb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+723:3628:x1870152993236096:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.130159:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026583 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.130160:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36386 00010000:00010000:1.0:1783520936.130180:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026583 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.130188:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026583 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.130193:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026583 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.130194:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2030 count 1 00010000:00010000:1.0:1783520936.130219:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.130223:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026591 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7ee:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.130236:0:12734:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000006b7649a/0xd5401cbabc021cdc lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2084 expref: 282 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.130245:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007a2b1775/0xd5401cbabc02295b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f304b expref: 282 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.130260:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x7ee:0x0].0x0 since=37s v=(0 0) 00000100:00100000:3.0:1783520936.130261:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516649500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993235072:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.130271:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ee:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.130281:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026591 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.130295:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02656e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.130307:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007a2b1775/0xd5401cbabc02295b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f304b expref: 281 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.130308:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02656e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.130345:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505875c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+724:3626:x1870152993235456:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2721us (2868us total) trans 4294976292 rc 0/0 00010000:00010000:3.0:1783520936.130349:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02656e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.130353:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007a2b1775/0xd5401cbabc02295b lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f304b expref: 281 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.130389:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44795 00010000:00010000:3.0:1783520936.130435:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02656e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.130442:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000004:00080000:1.0:1783520936.130444:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351664b100/2, rc 0, transno 4294976292 00000100:00100000:2.0:1783520936.130445:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032e8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993236224:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.130448:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026583 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.130464:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.130471:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026583 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.130475:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351664b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993235456:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.130500:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026583 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.130532:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026583 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.130544:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57d80 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.130557:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:0.0:1783520936.130570:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026583 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.130573:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993236352:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.130608:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.130613:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.130619:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.130626:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.130630:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02656e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.130653:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.130671:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993236352 00010000:00010000:3.0:1783520936.130672:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.130681:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.130683:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993236352 00010000:00010000:3.0:1783520936.130684:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02656e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.130684:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02657c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.130702:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02657c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.130719:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02657c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.130719:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505877800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+281:2387:x1870153002460416:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 739us (871us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.130730:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44798 00010000:00010000:1.0:1783520936.130749:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02657c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.130751:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b21180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+724:3626:x1870152993236352:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.130751:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36387 00010000:00010000:3.0:1783520936.130776:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00002000:00080000:2.0:1783520936.130779:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2031 count 1 00010000:00010000:3.0:1783520936.130784:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02656e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.130790:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.130799:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.130800:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02656e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.130805:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02657c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.130835:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.130843:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02656e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.130844:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02657c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.130862:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026560 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.130886:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02656e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.130893:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02657c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.130897:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505876d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+725:3625:x1870152993235328:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3586us (3673us total) trans 4294976293 rc 0/0 00010000:00010000:0.0:1783520936.130902:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026560 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.130933:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02657c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.130941:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44794 00010000:00010000:0.0:1783520936.130964:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026560 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.130975:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02657c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.130991:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.130992:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026560 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.131001:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520936.131003:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516648380/2, rc 0, transno 4294976293 00010000:00010000:3.0:1783520936.131008:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.131010:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.131018:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.131021:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.131028:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516648380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993235328:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.131040:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026598 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7ef:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.131049:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.131054:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007a20b8f0/0xd5401cbabc021d0d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f20d8 expref: 280 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.131063:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ef:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.131067:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ef:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.131071:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026598 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.131085:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.131089:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026560 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.131097:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.131102:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026560 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.131106:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:2.0:1783520936.131108:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026560 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.131124:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026560 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.131128:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993236480:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.131131:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026560 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.131153:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.131156:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.131158:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.131197:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+726:3625:x1870152993233920:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6393us (6644us total) trans 4294976295 rc 0/0 00000100:00100000:1.0:1783520936.131201:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:2.0:1783520936.131211:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516648000/2, rc 0, transno 4294976294 00000100:00100000:3.0:1783520936.131212:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351664bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+726:3628:x1870152993234816:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5347us (5466us total) trans 4294976294 rc 0/0 00000100:00100000:0.0:1783520936.131223:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44790 00000100:00100000:2.0:1783520936.131230:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516648000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993234816:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.131238:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44792 00000004:00080000:1.0:1783520936.131251:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513b21500/2, rc 0, transno 4294976295 00000100:00100000:0.0:1783520936.131263:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993236480 00000100:00100000:1.0:1783520936.131264:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993233920:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.131272:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993236480 00010000:00010000:2.0:1783520936.131290:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007a20b8f0/0xd5401cbabc021d0d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20d8 expref: 280 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.131304:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007a20b8f0/0xd5401cbabc021d0d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20d8 expref: 280 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.131320:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44799 00000100:00100000:2.0:1783520936.131321:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032eb800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993236608:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.131341:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b23480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+725:3625:x1870152993236480:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.131342:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993236864:0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.131439:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2032 count 1 00010000:00010000:0.0:1783520936.131460:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc02659f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7f0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.131476:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ecec65c0/0xd5401cbabc021d14 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f20e6 expref: 280 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.131484:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f0:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.131488:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.131503:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc02659f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.131521:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.131527:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607eb8180 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.131533:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ecec65c0/0xd5401cbabc021d14 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20e6 expref: 280 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.131547:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ecec65c0/0xd5401cbabc021d14 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20e6 expref: 280 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.131555:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993236864 00000100:00100000:1.0:1783520936.131556:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1783520936.131563:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993236864 00000100:00100000:1.0:1783520936.131570:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993236736:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.131583:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943505874000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993236992:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.131606:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44800 00000100:00100000:1.0:1783520936.131615:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.131621:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b20a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+726:3628:x1870152993236864:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.131632:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993236736 00000100:00100000:0.0:1783520936.131634:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895840 going to sleep for 11 seconds 00002000:00080000:3.0:1783520936.131640:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2034 count 1 00000100:00100000:2.0:1783520936.131642:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993236736 00010000:00010000:3.0:1783520936.131673:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0265a6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7f2:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.131694:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c9af1523/0xd5401cbabc021d22 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2102 expref: 280 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.131705:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f2:0x0].0x0 since=38s v=(0 0) 00000100:00100000:2.0:1783520936.131710:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44801 00010000:00010000:3.0:1783520936.131714:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.131721:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0265a6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.131730:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b21f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+727:3628:x1870152993236736:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.131757:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2033 count 1 00010000:00010000:2.0:1783520936.131777:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0265ad lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7f1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.131779:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c9af1523/0xd5401cbabc021d22 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2102 expref: 280 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.131792:0:20415:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008b1d41a4/0xd5401cbabc021cff lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20bc expref: 280 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.131814:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000ff0eacb/0xd5401cbabc021ce3 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2092 expref: 280 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.131820:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000bf70fde2/0xd5401cbabc021d1b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f20f4 expref: 280 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.131847:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c9af1523/0xd5401cbabc021d22 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2102 expref: 280 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.131890:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943505877480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993235200:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.131891:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f1:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.131901:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f1:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.131901:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b21880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993235968:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.131905:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503371500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993237120:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.131914:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0265ad lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.131916:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026575 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.131935:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02658a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.131946:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026575 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.131962:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6540 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.131985:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026575 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.132000:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02658a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.132001:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026575 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.132016:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000bf70fde2/0xd5401cbabc021d1b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20f4 expref: 278 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.132061:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02658a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.132062:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026575 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.132073:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000bf70fde2/0xd5401cbabc021d1b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20f4 expref: 278 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.132106:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02658a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.132123:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.132145:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.132154:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02658a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.132166:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032eaa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993237248:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.132179:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:0.0:1783520936.132182:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00000100:00100000:2.0:1783520936.132186:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002460544 00000100:00100000:2.0:1783520936.132190:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002460544 00010000:00010000:0.0:1783520936.132192:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026575 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.132204:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.132211:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.132213:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026575 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.132219:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02658a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.132222:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007a2b1775/0xd5401cbabc02295b lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x7ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f304b expref: 279 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.132250:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026575 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.132283:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026575 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.132286:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351664aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+728:3628:x1870152993235584:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4383us (4592us total) trans 4294976296 rc 0/0 00000100:00100000:2.0:1783520936.132299:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36388 00010000:00010000:3.0:1783520936.132307:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb600 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520936.132322:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503370380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+279:2387:x1870153002460544:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.132322:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44796 00010000:00010000:3.0:1783520936.132323:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02658a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.132323:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026575 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.132342:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.132400:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02658a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.132407:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007a2b1775/0xd5401cbabc02295b lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x7ee:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f304b expref: 279 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.132416:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516649880/2, rc 0, transno 4294976296 00010000:00010000:0.0:1783520936.132421:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.132429:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.132439:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516649880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993235584:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.132444:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02658a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.132444:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.132466:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc02658a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.132470:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007a2b1775/0xd5401cbabc02295b lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7ee:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f304b expref: 279 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.132479:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026567 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.132521:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993237376:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.132548:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ee:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.132551:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026567 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.132552:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ee:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.132559:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026591 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ee:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.132570:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026591 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7ee:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.132570:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026567 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.132592:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.132599:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.132607:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.132617:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.132624:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026567 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.132655:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.132670:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520936.132673:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993237376 00010000:00010000:2.0:1783520936.132677:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026567 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.132681:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993237376 00010000:00010000:2.0:1783520936.132693:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.132699:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026567 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.132709:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026567 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.132717:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44802 00010000:00010000:2.0:1783520936.132732:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026567 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.132733:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b23b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+729:3625:x1870152993237376:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.132756:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026567 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.132764:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2035 count 1 00010000:00010000:2.0:1783520936.132780:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.132784:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.132789:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0265b4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7f3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.132790:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.132790:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+729:3625:x1870152993234304:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7593us (7700us total) trans 4294976297 rc 0/0 00010000:00010000:2.0:1783520936.132809:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.132821:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006519a761/0xd5401cbabc021d30 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2110 expref: 279 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.132822:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44791 00010000:00010000:1.0:1783520936.132836:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f3:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.132839:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.132843:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0265b4 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.132852:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503370380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+279:2387:x1870153002460544:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 539us (742us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.132863:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36388 00010000:00010000:1.0:1783520936.132870:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006519a761/0xd5401cbabc021d30 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2110 expref: 279 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.132878:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516649180/2, rc 0, transno 4294976297 00010000:00010000:1.0:1783520936.132887:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006519a761/0xd5401cbabc021d30 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2110 expref: 278 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.132899:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516649180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993234304:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.132909:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b20380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993237504:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.132913:0:8410:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007a2b1775/0xd5401cbabc02295b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f304b expref: 278 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.132935:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032e8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993236224:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.132961:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026591 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.132968:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470240 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.132973:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026591 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.132985:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026591 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.132995:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943505877480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993237632:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.132996:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026591 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.133015:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026591 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.133036:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.133048:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.133065:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.133078:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026591 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.133090:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.133096:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026591 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.133110:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993237632 00010000:00010000:3.0:1783520936.133111:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026591 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.133116:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993237632 00000100:00100000:1.0:1783520936.133147:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44803 00000100:00100000:2.0:1783520936.133181:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032eb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+731:3628:x1870152993236096:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3032us (3201us total) trans 4294976298 rc 0/0 00000004:00080000:1.0:1783520936.133182:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351664ad80/2, rc 0, transno 4294976298 00000100:00100000:1.0:1783520936.133201:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351664ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993236096:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.133202:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44797 00010000:00010000:3.0:1783520936.133232:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026591 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.133241:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026591 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.133252:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351664b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+730:3627:x1870152993234944:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6978us (7089us total) trans 4294976299 rc 0/0 00000100:00100000:0.0:1783520936.133269:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44793 00010000:00010000:3.0:1783520936.133279:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.133284:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.133288:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.133296:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993237760:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.133300:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607eb8300 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.133348:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:1.0:1783520936.133429:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516648e00/2, rc 0, transno 4294976299 00000100:00100000:1.0:1783520936.133449:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516648e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993234944:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.133463:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993237760 00000100:00100000:2.0:1783520936.133474:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993237760 00000100:00100000:1.0:1783520936.133560:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1783520936.133568:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44804 00000100:00100000:1.0:1783520936.133575:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cc7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993237888:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.133588:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943505874000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993236992:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.133589:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cc7d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+730:3627:x1870152993237760:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.133610:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.133619:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00080000:2.0:1783520936.133636:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2037 count 1 00010000:00010000:0.0:1783520936.133637:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc02659f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.133660:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0265bb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7f5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.133669:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993237888 00000100:00100000:1.0:1783520936.133677:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993237888 00010000:00010000:2.0:1783520936.133678:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006b5c6fc3/0xd5401cbabc021d3e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f212c expref: 277 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.133691:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f5:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.133694:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.133698:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0265bb lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.133722:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44805 00010000:00010000:2.0:1783520936.133734:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006b5c6fc3/0xd5401cbabc021d3e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f212c expref: 277 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.133743:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cc7d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+731:3627:x1870152993237888:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.133773:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006b5c6fc3/0xd5401cbabc021d3e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f212c expref: 277 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.133778:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2038 count 1 00000100:00100000:2.0:1783520936.133789:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032e8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993238016:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.133796:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0265c2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7f6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.133813:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000be102d1b/0xd5401cbabc022bb5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f32a5 expref: 277 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.133821:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x7f6:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.133825:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.133831:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0265c2 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.133851:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f180 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.133861:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000be102d1b/0xd5401cbabc022bb5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f32a5 expref: 277 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.133877:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000be102d1b/0xd5401cbabc022bb5 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f32a5 expref: 277 pid: 35274 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.133892:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cc7ea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993238144:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.133902:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032eb800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993236608:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.133928:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026598 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.133933:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124700c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.133962:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b22d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+731:3626:x1870152993237632:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.133978:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2036 count 1 00010000:00010000:1.0:1783520936.133993:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0265c9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7f4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.134008:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000fdb670bb/0xd5401cbabc021d37 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f211e expref: 277 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.134017:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f4:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.134019:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.134022:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0265c9 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.134047:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000fdb670bb/0xd5401cbabc021d37 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f211e expref: 277 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.134059:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000fdb670bb/0xd5401cbabc021d37 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f211e expref: 277 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.134068:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e0f4e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993238272:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.134089:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470000 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.134161:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.134207:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002460800 00000100:00100000:1.0:1783520936.134212:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002460800 00010000:00010000:1.0:1783520936.134237:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ecec65c0/0xd5401cbabc021d14 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20e6 expref: 278 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.134256:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002460672 00000100:00100000:1.0:1783520936.134258:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002460672 00010000:00010000:1.0:1783520936.134266:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007a20b8f0/0xd5401cbabc021d0d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20d8 expref: 279 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.134282:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36389 00000100:00100000:1.0:1783520936.134294:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503373480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+279:2389:x1870153002460800:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.134306:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36390 00010000:00010000:1.0:1783520936.134307:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.134308:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503371500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993237120:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.134324:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ecec65c0/0xd5401cbabc021d14 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7f0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20e6 expref: 279 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.134331:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503372a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+279:2389:x1870153002460672:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.134388:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.134389:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ecec65c0/0xd5401cbabc021d14 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7f0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20e6 expref: 279 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.134403:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007a20b8f0/0xd5401cbabc021d0d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ef:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20d8 expref: 279 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.134415:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0265a6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.134432:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.134469:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ecec65c0/0xd5401cbabc021d14 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7f0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20e6 expref: 279 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.134470:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007a20b8f0/0xd5401cbabc021d0d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7ef:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20d8 expref: 279 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.134476:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002460928 00000100:00100000:3.0:1783520936.134485:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002460928 00000100:00100000:2.0:1783520936.134517:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.134528:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.134529:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c9af1523/0xd5401cbabc021d22 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2102 expref: 279 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.134538:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f0:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.134542:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002461056 00010000:00010000:0.0:1783520936.134542:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007a20b8f0/0xd5401cbabc021d0d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7ef:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20d8 expref: 279 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.134551:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002461056 00010000:00010000:1.0:1783520936.134562:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc02659f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.134568:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ef:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.134578:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006519a761/0xd5401cbabc021d30 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2110 expref: 279 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.134589:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36391 00010000:00010000:0.0:1783520936.134590:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ef:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.134597:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc02659f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.134604:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026598 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.134611:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032eb800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+279:2389:x1870153002460928:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.134638:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.134651:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36392 00010000:00010000:3.0:1783520936.134652:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c9af1523/0xd5401cbabc021d22 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7f2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2102 expref: 279 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.134668:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638586050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+279:2388:x1870153002461056:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.134670:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026598 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.134677:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c9af1523/0xd5401cbabc021d22 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7f2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2102 expref: 279 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.134681:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.134688:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006519a761/0xd5401cbabc021d30 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7f3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2110 expref: 279 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.134690:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.134705:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c9af1523/0xd5401cbabc021d22 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7f2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2102 expref: 279 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.134708:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006519a761/0xd5401cbabc021d30 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7f3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2110 expref: 279 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.134717:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.134721:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.134723:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.134726:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.134726:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.134734:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0265a6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.134734:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0265b4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.134747:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503373480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+278:2389:x1870153002460800:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 449us (1147us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.134758:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0265a6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.134758:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0265b4 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7f3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.134766:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503372a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+278:2389:x1870153002460672:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 434us (599us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.134787:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36389 00000100:00100000:0.0:1783520936.134800:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36390 00010000:00010000:2.0:1783520936.134815:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.134819:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.134841:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638586050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+276:2388:x1870153002461056:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 174us (349us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.134850:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032eb800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+276:2389:x1870153002460928:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 246us (433us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.134850:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36392 00000100:00100000:3.0:1783520936.134857:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36391 00010000:00010000:1.0:1783520936.134865:0:35276:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006519a761/0xd5401cbabc021d30 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2110 expref: 275 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.134889:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b20380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993237504:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.134896:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026598 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.134908:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026598 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.134918:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026598 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.134921:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0265b4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.134928:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0265a6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.134938:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026598 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.134966:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.134975:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0265b4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.134993:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0265b4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.134994:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.135002:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.135014:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026598 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.135024:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0265b4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.135026:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.135036:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026598 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.135062:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0265b4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.135072:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0265a6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.135073:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026598 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.135117:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026598 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.135127:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b21180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+732:3626:x1870152993236352:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4378us (4505us total) trans 4294976300 rc 0/0 00010000:00010000:0.0:1783520936.135128:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026598 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.135144:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0265a6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.135144:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44798 00010000:00010000:3.0:1783520936.135154:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0265a6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.135156:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513b20000/2, rc 0, transno 4294976300 00010000:00010000:0.0:1783520936.135160:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.135164:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.135174:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.135184:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993236352:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.135186:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002461184 00010000:00010000:3.0:1783520936.135187:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.135196:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002461184 00010000:00010000:3.0:1783520936.135213:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.135227:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0265a6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.135229:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000bf70fde2/0xd5401cbabc021d1b lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20f4 expref: 274 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.135237:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.135244:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0265a6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.135278:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36393 00000100:00100000:2.0:1783520936.135289:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505874000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+274:2388:x1870153002461184:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.135294:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e0f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993238400:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.135302:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.135306:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0265a6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.135310:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000bf70fde2/0xd5401cbabc021d1b lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7f1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20f4 expref: 274 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.135325:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000bf70fde2/0xd5401cbabc021d1b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7f1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20f4 expref: 274 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.135341:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.135345:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.135351:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0265ad lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.135383:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0265ad lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7f1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.135398:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0265a6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.135398:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.135406:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.135419:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.135425:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc0265a6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.135425:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505874000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+274:2388:x1870153002461184:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 138us (472us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.135434:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36393 00010000:00010000:3.0:1783520936.135453:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.135455:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.135455:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002461312 00010000:00010000:3.0:1783520936.135458:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.135460:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002461312 00010000:00010000:2.0:1783520936.135484:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006b5c6fc3/0xd5401cbabc021d3e lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f212c expref: 274 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.135530:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36394 00010000:00010000:1.0:1783520936.135533:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.135542:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505875f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+274:2388:x1870153002461312:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.135544:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.135550:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0265b4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.135556:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.135556:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.135562:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0265b4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.135566:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006b5c6fc3/0xd5401cbabc021d3e lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7f5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f212c expref: 274 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.135575:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0265b4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.135585:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006b5c6fc3/0xd5401cbabc021d3e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7f5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f212c expref: 274 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.135591:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0265b4 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.135599:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.135603:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc0265b4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.135604:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.135610:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0265bb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.135622:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0265bb lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7f5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.135640:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.135642:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.135648:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.135653:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.135663:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505875f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+274:2388:x1870153002461312:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 125us (306us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.135669:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36394 00010000:00010000:2.0:1783520936.135691:0:8410:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006b5c6fc3/0xd5401cbabc021d3e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f212c expref: 273 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.135710:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032e8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993238016:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.135718:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b23b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+733:3625:x1870152993237376:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2983us (3136us total) trans 4294976301 rc 0/0 00000100:00100000:1.0:1783520936.135732:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44802 00010000:00010000:2.0:1783520936.135739:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0265bb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.135744:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.135756:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0265bb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.135766:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0265bb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.135775:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0265bb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.135779:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b20a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+732:3628:x1870152993236864:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4161us (4260us total) trans 4294976302 rc 0/0 00010000:00010000:2.0:1783520936.135784:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0265bb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.135787:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44800 00000004:00080000:1.0:1783520936.135792:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513b21880/2, rc 0, transno 4294976301 00010000:00010000:2.0:1783520936.135802:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.135809:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993237376:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.135819:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.135828:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0265bb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.135838:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.135845:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0265bb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.135845:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.135856:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0265bb lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.135903:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1783520936.135904:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993238400 00000100:00100000:0.0:1783520936.135917:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993238400 00000100:00100000:1.0:1783520936.135925:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e0f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993238528:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.135927:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0265bb lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.135927:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cc7d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+733:3627:x1870152993237760:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2344us (2576us total) trans 4294976303 rc 0/0 00010000:00010000:3.0:1783520936.135946:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0265bb lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.135948:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44804 00000100:00100000:0.0:1783520936.135955:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44806 00010000:00010000:3.0:1783520936.135965:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.135966:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e0f6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+732:3625:x1870152993238400:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.135969:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.135972:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002461440 00010000:00010000:3.0:1783520936.135973:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.135975:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002461440 00010000:00010000:2.0:1783520936.135995:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000be102d1b/0xd5401cbabc022bb5 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x7f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f32a5 expref: 273 pid: 35274 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.136005:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513b22680/2, rc 0, transno 4294976302 00002000:00080000:0.0:1783520936.136009:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2039 count 1 00000100:00100000:2.0:1783520936.136015:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002461568 00000100:00100000:2.0:1783520936.136020:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002461568 00000100:00100000:3.0:1783520936.136024:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993236864:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.136028:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.136034:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0265d0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7f7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.136039:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000fdb670bb/0xd5401cbabc021d37 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f211e expref: 274 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.136056:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993238528 00010000:00010000:0.0:1783520936.136061:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000bdc77088/0xd5401cbabc021d53 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2148 expref: 274 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.136067:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f7:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.136071:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.136075:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0265d0 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.136090:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36395 00010000:00010000:0.0:1783520936.136101:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000bdc77088/0xd5401cbabc021d53 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2148 expref: 274 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.136102:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505876680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+274:2387:x1870153002461440:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.136117:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.136125:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993238528 00010000:00010000:2.0:1783520936.136126:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000be102d1b/0xd5401cbabc022bb5 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x7f6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f32a5 expref: 274 pid: 35274 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.136132:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000bdc77088/0xd5401cbabc021d53 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2148 expref: 274 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.136142:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e0f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993238784:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.136144:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000be102d1b/0xd5401cbabc022bb5 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7f6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f32a5 expref: 274 pid: 35274 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.136164:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943505877800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993238656:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.136178:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44807 00000100:00100000:3.0:1783520936.136188:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e0f6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+733:3625:x1870152993238528:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.136188:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.136192:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.136197:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0265c2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.136201:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2040 count 1 00010000:00010000:2.0:1783520936.136206:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0265c2 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7f6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.136213:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc0265d7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7f8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.136222:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.136224:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000044fb977c/0xd5401cbabc021d5a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2156 expref: 274 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.136234:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.136242:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895840 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.136243:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f8:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.136251:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f8:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.136251:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505876680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+274:2387:x1870153002461440:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 152us (512us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.136256:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc0265d7 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.136261:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36395 00000100:00100000:0.0:1783520936.136272:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36396 00000100:00100000:0.0:1783520936.136283:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505874e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+273:2388:x1870153002461568:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.136288:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000044fb977c/0xd5401cbabc021d5a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2156 expref: 273 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.136291:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513b20700/2, rc 0, transno 4294976303 00010000:00010000:0.0:1783520936.136293:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.136294:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993238784 00000100:00100000:2.0:1783520936.136300:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993238784 00010000:00010000:0.0:1783520936.136309:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000fdb670bb/0xd5401cbabc021d37 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x7f4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f211e expref: 273 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.136319:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993237760:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.136320:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000044fb977c/0xd5401cbabc021d5a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2156 expref: 273 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.136327:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000fdb670bb/0xd5401cbabc021d37 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7f4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f211e expref: 273 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.136339:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44808 00000100:00100000:2.0:1783520936.136354:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e0f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+734:3627:x1870152993238784:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.136388:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.136392:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.136396:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0265c9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.136402:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0265c9 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7f4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.136416:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.136430:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503370380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993238912:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.136437:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505874e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+273:2388:x1870153002461568:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 156us (597us total) trans 0 rc 0/0 00002000:00080000:2.0:1783520936.136444:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2041 count 1 00000100:00100000:0.0:1783520936.136448:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36396 00010000:00010000:2.0:1783520936.136461:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0265de lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7f9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.136476:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000cc7bb301/0xd5401cbabc021d61 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2164 expref: 272 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.136479:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993239040:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.136483:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f9:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.136488:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.136506:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0265de lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.136534:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000cc7bb301/0xd5401cbabc021d61 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2164 expref: 272 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.136545:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.136546:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000cc7bb301/0xd5401cbabc021d61 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2164 expref: 272 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.136562:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032e8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993239168:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.136587:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000be102d1b/0xd5401cbabc022bb5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f32a5 expref: 272 pid: 35274 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.136614:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cc7ea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993238144:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.136616:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6c00 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.136616:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607eb8180 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.136647:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993239040 00000100:00100000:2.0:1783520936.136655:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993239040 00010000:00010000:1.0:1783520936.136656:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0265c2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.136663:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc02659f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.136679:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0265c2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.136684:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc02659f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.136691:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44809 00010000:00010000:1.0:1783520936.136705:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0265c2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.136711:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc02659f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.136712:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635784c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+735:3628:x1870152993239040:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.136744:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2042 count 1 00010000:00010000:1.0:1783520936.136755:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0265c2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.136756:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc02659f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.136762:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0265e5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7fa:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.136813:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0265c2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.136829:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000029646d1/0xd5401cbabc021d68 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2172 expref: 271 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.136831:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.136847:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520936.136848:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7fa:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.136848:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520936.136857:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.136861:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7fa:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.136863:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc02659f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.136864:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0265c2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.136865:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0265e5 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.136889:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.136889:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbebe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.136898:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0265c2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.136898:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc02659f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.136904:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000029646d1/0xd5401cbabc021d68 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2172 expref: 271 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.136919:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc02659f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.136921:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0265c2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.136924:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000029646d1/0xd5401cbabc021d68 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2172 expref: 271 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.136949:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0265c2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.136952:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032eb800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993239296:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.136958:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc0265c2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.136959:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc02659f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.136977:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc02659f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.136985:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.136985:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cc7d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+736:3627:x1870152993237888:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3244us (3369us total) trans 4294976304 rc 0/0 00010000:00010000:3.0:1783520936.136994:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.136997:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f180 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.136999:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.136999:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.137006:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.137011:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.137022:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44805 00010000:00010000:2.0:1783520936.137057:0:12734:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000bf70fde2/0xd5401cbabc021d1b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f20f4 expref: 271 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.137079:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032eaa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993237248:192.168.204.33@tcp:104: 00000004:00080000:1.0:1783520936.137089:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cc7c380/2, rc 0, transno 4294976304 00010000:00010000:2.0:1783520936.137097:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0265ad lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.137106:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cc7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993237888:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.137113:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0265ad lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.137126:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0265ad lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.137133:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0265ad lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.137139:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0265ad lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.137165:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.137178:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cc7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993239424:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.137196:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b23480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+736:3625:x1870152993236480:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5862us (6005us total) trans 4294976305 rc 0/0 00000100:00100000:0.0:1783520936.137207:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44799 00010000:00010000:3.0:1783520936.137216:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.137222:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0265ad lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.137232:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.137244:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.137247:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0265ad lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.137261:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0265ad lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.137269:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993239424 00000100:00100000:0.0:1783520936.137277:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993239424 00010000:00010000:3.0:1783520936.137278:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0265ad lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.137281:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b21f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+736:3628:x1870152993236736:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5557us (5669us total) trans 4294976306 rc 0/0 00010000:00010000:3.0:1783520936.137288:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc0265ad lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.137292:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44801 00000004:00080000:1.0:1783520936.137296:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513b23100/2, rc 0, transno 4294976305 00010000:00010000:3.0:1783520936.137307:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.137312:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993236480:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.137313:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.137317:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.137319:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44810 00000100:00100000:2.0:1783520936.137333:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+736:3625:x1870152993239424:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.137349:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2043 count 1 00000004:00080000:3.0:1783520936.137394:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513b22300/2, rc 0, transno 4294976306 00010000:00010000:2.0:1783520936.137396:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0265ec lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7fb:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.137410:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007c87fb5a/0xd5401cbabc021d6f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2180 expref: 270 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.137418:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7fb:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.137422:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7fb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.137428:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0265ec lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.137438:0:8408:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000fdb670bb/0xd5401cbabc021d37 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f211e expref: 270 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.137450:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1783520936.137454:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e0f4e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993238272:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.137459:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007c87fb5a/0xd5401cbabc021d6f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2180 expref: 269 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.137465:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993239552:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.137473:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993236736:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.137478:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0265c9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.137482:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:2.0:1783520936.137483:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007c87fb5a/0xd5401cbabc021d6f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2180 expref: 269 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.137511:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0265c9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.137520:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032eaa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993239808:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.137523:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0265c9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.137536:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.137537:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993239680:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.137557:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0265c9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.137568:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0265c9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.137598:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520936.137599:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.137606:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607eb8300 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.137648:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.137654:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0265c9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.137658:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993239552 00010000:00010000:3.0:1783520936.137661:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbaa00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520936.137663:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993239552 00010000:00010000:3.0:1783520936.137668:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0265c9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.137677:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993239680 00010000:00010000:3.0:1783520936.137685:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0265c9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.137687:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993239680 00000100:00100000:2.0:1783520936.137704:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44811 00010000:00010000:3.0:1783520936.137708:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0265c9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.137715:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0265c9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.137715:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+739:3625:x1870152993239552:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.137730:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2044 count 1 00000100:00100000:0.0:1783520936.137742:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b22d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+739:3626:x1870152993237632:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4583us (4689us total) trans 4294976307 rc 0/0 00010000:00010000:3.0:1783520936.137747:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.137747:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0265f3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7fc:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.137755:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.137758:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44803 00010000:00010000:3.0:1783520936.137759:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.137760:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000085fcf576/0xd5401cbabc021d84 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f218e expref: 269 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.137768:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943505877480/2, rc 0, transno 4294976307 00010000:00010000:2.0:1783520936.137769:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7fc:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.137780:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7fc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.137785:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0265f3 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.137792:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943505877480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993237632:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.137801:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44812 00010000:00010000:2.0:1783520936.137817:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000085fcf576/0xd5401cbabc021d84 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f218e expref: 269 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.137818:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503373480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+738:3628:x1870152993239680:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.137836:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000085fcf576/0xd5401cbabc021d84 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f218e expref: 269 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.137842:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2045 count 1 00000100:00100000:2.0:1783520936.137857:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc0380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993239936:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.137860:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0265fa lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7fd:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.137867:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:0.0:1783520936.137877:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000221c3757/0xd5401cbabc021d8b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f219c expref: 269 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.137882:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993240064:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.137891:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57d80 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.137900:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7fd:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.137905:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7fd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.137909:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0265fa lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.137937:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000221c3757/0xd5401cbabc021d8b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f219c expref: 269 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.137957:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.137962:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000221c3757/0xd5401cbabc021d8b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f219c expref: 269 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.137976:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e0f4e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993240192:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.138023:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895600 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.138035:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993240064 00000100:00100000:1.0:1783520936.138040:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993240064 00000100:00100000:2.0:1783520936.138063:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032e8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993239168:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.138076:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44813 00000100:00100000:1.0:1783520936.138089:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+739:3627:x1870152993240064:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.138094:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0265de lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.138108:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2046 count 1 00010000:00010000:1.0:1783520936.138128:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026601 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7fe:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.138137:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032eb800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993239296:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.138167:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0265e5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.138169:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000842c8147/0xd5401cbabc022aa4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7fe:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3194 expref: 269 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.138199:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503370380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993238912:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.138208:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x7fe:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.138217:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7fe:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.138222:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc0265d7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.138224:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026601 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7fe:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.138244:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.138274:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.138279:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943505877800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993238656:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.138290:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000842c8147/0xd5401cbabc022aa4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7fe:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3194 expref: 269 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.138295:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0265d0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.138314:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.138315:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002461824 00010000:00010000:1.0:1783520936.138322:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000842c8147/0xd5401cbabc022aa4 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7fe:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3194 expref: 269 pid: 8410 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.138325:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002461824 00000100:00100000:1.0:1783520936.138348:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c551c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993240320:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.138403:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.138405:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000cc7bb301/0xd5401cbabc021d61 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2164 expref: 270 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.138445:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002461696 00000100:00100000:0.0:1783520936.138446:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002461952 00000100:00100000:2.0:1783520936.138450:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002461696 00000100:00100000:0.0:1783520936.138452:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002461952 00010000:00010000:2.0:1783520936.138476:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000bdc77088/0xd5401cbabc021d53 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2148 expref: 272 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.138482:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000044fb977c/0xd5401cbabc021d5a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2156 expref: 272 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.138483:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470a80 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.138524:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002462080 00000100:00100000:2.0:1783520936.138530:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002462080 00000100:00100000:0.0:1783520936.138542:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36397 00010000:00010000:2.0:1783520936.138550:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000029646d1/0xd5401cbabc021d68 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2172 expref: 273 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.138559:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36398 00000100:00100000:0.0:1783520936.138561:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513bc1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+273:2390:x1870153002461824:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.138584:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513bc3100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+273:2389:x1870153002461696:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.138587:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.138594:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36399 00010000:00010000:1.0:1783520936.138600:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.138600:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000cc7bb301/0xd5401cbabc021d61 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7f9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2164 expref: 273 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.138609:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943603c1ec50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+273:2389:x1870153002461952:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.138617:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000bdc77088/0xd5401cbabc021d53 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7f7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2148 expref: 273 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.138645:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000bdc77088/0xd5401cbabc021d53 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7f7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2148 expref: 273 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.138645:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000cc7bb301/0xd5401cbabc021d61 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7f9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2164 expref: 273 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.138651:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.138670:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000044fb977c/0xd5401cbabc021d5a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7f8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2156 expref: 273 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.138712:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000bdc77088/0xd5401cbabc021d53 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7f7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2148 expref: 273 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.138714:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000044fb977c/0xd5401cbabc021d5a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7f8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2156 expref: 273 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.138726:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cc7bb301/0xd5401cbabc021d61 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7f9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2164 expref: 273 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.138756:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.138762:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.138765:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.138769:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0265d0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.138770:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000044fb977c/0xd5401cbabc021d5a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7f8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2156 expref: 271 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.138773:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.138793:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0265de lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.138808:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0265d0 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.138821:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.138828:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0265de lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.138842:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7f8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.138854:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc0265d7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.138865:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc0265d7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.138869:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.138874:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.138894:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.138907:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc0265d7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.138918:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc0265d7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.138925:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513bc3100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+270:2389:x1870153002461696:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 338us (642us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.138926:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943603c1ec50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+270:2389:x1870153002461952:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 322us (619us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.138926:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513bc1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+270:2390:x1870153002461824:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 363us (678us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.138963:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36398 00000100:00100000:2.0:1783520936.138975:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36399 00000100:00100000:0.0:1783520936.138983:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36397 00000100:00100000:1.0:1783520936.138999:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36400 00000100:00100000:1.0:1783520936.139014:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513bc3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+267:2387:x1870153002462080:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.139018:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0265de lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.139026:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0265de lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.139026:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.139040:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000029646d1/0xd5401cbabc021d68 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7fa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2172 expref: 267 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.139041:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0265de lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.139074:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e0f4e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993240192:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.139083:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0265de lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.139109:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000029646d1/0xd5401cbabc021d68 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7fa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2172 expref: 267 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.139112:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.139113:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0265fa lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.139185:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000029646d1/0xd5401cbabc021d68 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7fa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2172 expref: 267 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.139201:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7fa:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.139205:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7fa:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.139210:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0265e5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.139215:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e0f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+740:3627:x1870152993238784:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2861us (2986us total) trans 4294976308 rc 0/0 00000004:00080000:0.0:1783520936.139224:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e0f7100/2, rc 0, transno 4294976308 00010000:00010000:1.0:1783520936.139240:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0265e5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.139242:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44808 00000100:00100000:0.0:1783520936.139247:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e0f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993238784:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.139285:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.139292:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc0380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993239936:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.139314:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0265f3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.139327:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc0265d7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.139332:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513bc3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+266:2387:x1870153002462080:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 317us (978us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.139344:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e0f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993240448:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.139345:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0265e5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.139377:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc0265d7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.139390:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.139400:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0265de lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.139407:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36400 00010000:00010000:2.0:1783520936.139416:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0265e5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.139430:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.139433:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0265e5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.139436:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0265de lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.139449:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0265e5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.139450:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.139455:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0265de lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.139460:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0265d0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.139475:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.139512:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0265de lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.139512:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0265d0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.139513:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.139527:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0265e5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.139563:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0265de lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.139568:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635784c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+740:3628:x1870152993239040:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2855us (3020us total) trans 4294976309 rc 0/0 00010000:00010000:0.0:1783520936.139584:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.139590:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0265d0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.139593:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0265e5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.139594:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44809 00000100:00100000:2.0:1783520936.139612:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993240448 00000100:00100000:2.0:1783520936.139620:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993240448 00010000:00010000:3.0:1783520936.139626:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.139632:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.139642:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0265d0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.139643:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.139648:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0265e5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.139653:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44814 00010000:00010000:3.0:1783520936.139663:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.139679:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+740:3625:x1870152993240448:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.139682:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.139695:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc0265d7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.139716:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.139718:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0265e5 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.139720:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2047 count 1 00010000:00010000:3.0:1783520936.139721:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc0265d7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.139736:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0265e5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.139742:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc0265d7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.139742:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026608 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7ff:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.139753:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.139757:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520936.139759:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513b20380/2, rc 0, transno 4294976309 00010000:00010000:0.0:1783520936.139761:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.139769:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b4098e11/0xd5401cbabc021d99 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x7ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f21b8 expref: 265 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.139778:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ff:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.139779:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc0265d7 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.139782:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ff:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.139783:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993239040:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.139801:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026608 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x7ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.139812:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc0265d7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.139835:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.139838:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.139841:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.139842:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b4098e11/0xd5401cbabc021d99 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21b8 expref: 265 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.139852:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b4098e11/0xd5401cbabc021d99 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21b8 expref: 265 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.139869:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc0380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993240704:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.139870:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c551500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993240576:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.139890:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.139894:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.139899:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0265d0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.139905:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.139909:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0265d0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.139912:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57d80 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.139921:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0265d0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.139922:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.139934:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0265d0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.139941:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc0265d0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.139948:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032eaa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993239808:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.139960:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.139965:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.139973:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0265ec lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.139974:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.139977:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993240576 00000100:00100000:1.0:1783520936.139988:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993240576 00000100:00100000:1.0:1783520936.140032:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44815 00000100:00100000:1.0:1783520936.140044:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+743:3626:x1870152993240576:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.140060:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e0f6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+743:3625:x1870152993238528:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3870us (4043us total) trans 4294976310 rc 0/0 00000100:00100000:3.0:1783520936.140068:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44807 00000004:00080000:1.0:1783520936.140084:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e0f5880/2, rc 0, transno 4294976310 00000100:00100000:0.0:1783520936.140087:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e0f6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+742:3625:x1870152993238400:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4120us (4730us total) trans 4294976311 rc 0/0 00000100:00100000:1.0:1783520936.140102:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e0f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993238528:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.140103:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44806 00000100:00100000:3.0:1783520936.140123:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000004:00080000:1.0:1783520936.140133:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e0f4380/2, rc 0, transno 4294976311 00000100:00100000:1.0:1783520936.140142:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e0f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993238400:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.140189:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993240960:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.140205:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.140236:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.140271:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993240960 00000100:00100000:3.0:1783520936.140274:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.140279:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993240960 00000100:00100000:1.0:1783520936.140285:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1783520936.140302:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993240832:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.140318:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44816 00000100:00100000:2.0:1783520936.140328:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c550000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+742:3628:x1870152993240960:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.140340:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2050 count 1 00010000:00010000:2.0:1783520936.140353:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02660f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x802:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.140380:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.140399:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002462208 00000100:00100000:0.0:1783520936.140405:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002462208 00010000:00010000:2.0:1783520936.140420:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c2e859c2/0xd5401cbabc021dae lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x802:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f21e2 expref: 266 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.140426:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x802:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.140429:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x802:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.140434:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007c87fb5a/0xd5401cbabc021d6f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2180 expref: 266 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.140436:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.140436:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02660f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x802:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.140447:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993240832 00000100:00100000:1.0:1783520936.140458:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993240832 00000100:00100000:0.0:1783520936.140482:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002462336 00000100:00100000:0.0:1783520936.140488:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002462336 00010000:00010000:2.0:1783520936.140515:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c2e859c2/0xd5401cbabc021dae lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x802:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21e2 expref: 267 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.140520:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000085fcf576/0xd5401cbabc021d84 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f218e expref: 267 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.140526:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44817 00000100:00100000:1.0:1783520936.140543:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c553100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+743:3628:x1870152993240832:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.140550:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c2e859c2/0xd5401cbabc021dae lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x802:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21e2 expref: 267 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.140565:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2049 count 1 00000100:00100000:2.0:1783520936.140572:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993241088:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.140583:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002462464 00010000:00010000:1.0:1783520936.140584:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026616 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x801:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.140593:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002462464 00000100:00100000:3.0:1783520936.140601:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002462592 00000100:00100000:3.0:1783520936.140611:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002462592 00010000:00010000:1.0:1783520936.140619:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000944e9d51/0xd5401cbabc021da7 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x801:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f21d4 expref: 268 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.140621:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000221c3757/0xd5401cbabc021d8b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f219c expref: 269 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.140628:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607eb8300 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.140635:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000842c8147/0xd5401cbabc022aa4 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x7fe:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3194 expref: 269 pid: 8410 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.140651:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x801:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.140673:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x801:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.140680:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026616 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x801:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.140705:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36401 00010000:00010000:1.0:1783520936.140710:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000944e9d51/0xd5401cbabc021da7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x801:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21d4 expref: 269 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.140723:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36402 00010000:00010000:1.0:1783520936.140726:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000944e9d51/0xd5401cbabc021da7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x801:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21d4 expref: 269 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.140732:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503370380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+269:2388:x1870153002462208:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.140747:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36403 00000100:00100000:0.0:1783520936.140748:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503372a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+269:2390:x1870153002462336:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.140758:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350d164000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993241216:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.140766:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.140771:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+269:2388:x1870153002462464:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.140772:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.140778:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007c87fb5a/0xd5401cbabc021d6f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7fb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2180 expref: 269 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.140784:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000085fcf576/0xd5401cbabc021d84 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7fc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f218e expref: 269 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.140787:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.140794:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124700c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.140801:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000221c3757/0xd5401cbabc021d8b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7fd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f219c expref: 269 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.140806:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007c87fb5a/0xd5401cbabc021d6f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7fb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2180 expref: 269 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.140819:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000085fcf576/0xd5401cbabc021d84 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7fc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f218e expref: 269 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.140820:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2048 count 1 00010000:00010000:2.0:1783520936.140846:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000221c3757/0xd5401cbabc021d8b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7fd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f219c expref: 269 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.140850:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02661d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x800:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.140856:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007c87fb5a/0xd5401cbabc021d6f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7fb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2180 expref: 269 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.140859:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000085fcf576/0xd5401cbabc021d84 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7fc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f218e expref: 269 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.140897:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000221c3757/0xd5401cbabc021d8b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7fd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f219c expref: 269 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.140916:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7fb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.140921:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000710fd570/0xd5401cbabc021da0 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x800:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f21c6 expref: 268 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.140925:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7fb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.140928:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7fc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.140938:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7fd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.140939:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0265ec lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.140947:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7fc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.140962:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7fd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.140975:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0265f3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.140987:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0265fa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.140994:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x800:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.140995:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0265ec lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.141014:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x800:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.141030:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0265f3 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.141037:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02661d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x800:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.141042:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0265fa lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.141052:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.141084:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503370380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+266:2388:x1870153002462208:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 357us (960us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.141093:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36401 00010000:00010000:0.0:1783520936.141095:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.141111:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.141114:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36404 00010000:00010000:1.0:1783520936.141121:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000710fd570/0xd5401cbabc021da0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x800:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21c6 expref: 265 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.141127:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+265:2387:x1870153002462592:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.141141:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503372a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+265:2390:x1870153002462336:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 394us (928us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.141145:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+265:2388:x1870153002462464:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 376us (872us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.141152:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.141163:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000842c8147/0xd5401cbabc022aa4 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x7fe:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3194 expref: 265 pid: 8410 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.141185:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000842c8147/0xd5401cbabc022aa4 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x7fe:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3194 expref: 265 pid: 8410 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.141185:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000710fd570/0xd5401cbabc021da0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x800:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21c6 expref: 265 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.141186:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36403 00000100:00100000:0.0:1783520936.141211:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36402 00010000:00010000:2.0:1783520936.141222:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0265f3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.141237:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7fe:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.141250:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7fe:0x0].0x0 since=0s v=(1 0) 00000100:00100000:1.0:1783520936.141253:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350d165880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993241344:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.141259:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0265fa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.141265:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0265f3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.141268:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026601 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7fe:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.141293:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0265fa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.141297:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026601 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7fe:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.141300:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0265f3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.141304:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0265fa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.141308:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470240 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.141315:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.141332:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0265f3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.141334:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0265fa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.141341:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+263:2387:x1870153002462592:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 216us (907us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.141427:0:35275:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000842c8147/0xd5401cbabc022aa4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7fe:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3194 expref: 263 pid: 8410 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.141443:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36404 00000100:00100000:1.0:1783520936.141448:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c551c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993240320:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.141457:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:0.0:1783520936.141467:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520936.141479:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1783520936.141480:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026601 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.141482:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.141500:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0265f3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.141504:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0265fa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.141527:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026601 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.141545:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.141558:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0265f3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.141580:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026601 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.141605:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.141608:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0265f3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.141608:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+744:3625:x1870152993239552:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3890us (4078us total) trans 4294976312 rc 0/0 00010000:00010000:0.0:1783520936.141617:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0265fa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.141641:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44811 00010000:00010000:3.0:1783520936.141667:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0265f3 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.141676:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0265ec lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.141677:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0265fa lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.141690:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026601 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.141707:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0265f3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.141725:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0265fa lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.141728:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0265ec lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.141751:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026601 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.141769:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.141777:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.141782:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0265fa lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.141785:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.141791:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0265ec lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.141823:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0265ec lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.141844:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351530d180/2, rc 0, transno 4294976312 00010000:00010000:2.0:1783520936.141848:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.141856:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.141860:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.141864:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0265ec lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.141866:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993239552:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.141869:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.141876:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.141889:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520936.141929:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+744:3625:x1870152993239424:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4597us (4701us total) trans 4294976313 rc 0/0 00000100:00100000:2.0:1783520936.141938:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44810 00000100:00100000:1.0:1783520936.141966:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350d165f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993241472:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.142016:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.142030:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0265ec lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.142042:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0265ec lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.142045:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503373480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+744:3628:x1870152993239680:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4228us (4449us total) trans 4294976314 rc 0/0 00000100:00100000:0.0:1783520936.142063:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44812 00000004:00080000:1.0:1783520936.142076:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cc7ea00/2, rc 0, transno 4294976313 00000100:00100000:0.0:1783520936.142093:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993241472 00000100:00100000:1.0:1783520936.142095:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cc7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993239424:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.142102:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993241472 00010000:00010000:3.0:1783520936.142141:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0265ec lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.142148:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44818 00010000:00010000:3.0:1783520936.142158:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0265ec lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.142164:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350d166a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+744:3625:x1870152993241472:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.142187:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00002000:00080000:0.0:1783520936.142188:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2051 count 1 00010000:00010000:0.0:1783520936.142209:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026624 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x803:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.142213:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c551c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993241600:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.142244:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007ecb899c/0xd5401cbabc021db5 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x803:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f21f0 expref: 261 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.142255:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x803:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.142260:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x803:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.142262:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.142265:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026624 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x803:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.142266:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.142274:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.142280:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.142287:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc0380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993240704:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.142317:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007ecb899c/0xd5401cbabc021db5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x803:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21f0 expref: 261 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.142320:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026608 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x7ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.142351:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007ecb899c/0xd5401cbabc021db5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x803:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21f0 expref: 261 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.142406:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993241600 00000004:00080000:1.0:1783520936.142410:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351530f480/2, rc 0, transno 4294976314 00000100:00100000:0.0:1783520936.142410:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993241728:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.142412:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993241600 00000100:00100000:1.0:1783520936.142425:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993239680:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.142471:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44819 00000100:00100000:2.0:1783520936.142487:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638995180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+745:3625:x1870152993241600:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.142503:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895600 going to sleep for 11 seconds 00002000:00080000:2.0:1783520936.142517:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2052 count 1 00000100:00100000:1.0:1783520936.142528:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943638995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993241856:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.142533:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02662b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x804:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.142550:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000004ca5903/0xd5401cbabc021dbc lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x804:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f21fe expref: 261 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.142557:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x804:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.142562:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x804:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.142572:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.142576:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02662b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x804:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.142611:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993241856 00010000:00010000:2.0:1783520936.142612:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000004ca5903/0xd5401cbabc021dbc lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x804:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21fe expref: 261 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.142620:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993241856 00010000:00010000:2.0:1783520936.142636:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000004ca5903/0xd5401cbabc021dbc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x804:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21fe expref: 261 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.142653:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc0380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993241984:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.142666:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.142680:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44820 00010000:00010000:3.0:1783520936.142683:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.142692:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026601 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.142692:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638997800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+746:3627:x1870152993241856:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.142693:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607eb8180 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.142708:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.142714:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026601 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.142717:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2053 count 1 00010000:00010000:3.0:1783520936.142727:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026601 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.142735:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026632 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x805:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.142743:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026601 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.142755:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026601 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.142755:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000061799a66/0xd5401cbabc021dca lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x805:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f220c expref: 261 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.142778:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x805:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.142783:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x805:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.142787:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026632 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x805:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.142789:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.142794:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.142798:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.142817:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+747:3627:x1870152993240064:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4730us (4873us total) trans 4294976315 rc 0/0 00010000:00010000:0.0:1783520936.142820:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000061799a66/0xd5401cbabc021dca lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x805:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f220c expref: 261 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.142842:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44813 00010000:00010000:0.0:1783520936.142846:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000061799a66/0xd5401cbabc021dca lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x805:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f220c expref: 261 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.142849:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351530c000/2, rc 0, transno 4294976315 00000100:00100000:0.0:1783520936.142868:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993242112:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.142879:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993240064:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.142908:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895840 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.142948:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1783520936.142962:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350544ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993242240:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.142998:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.143004:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.143029:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993242240 00000100:00100000:0.0:1783520936.143033:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993242240 00000100:00100000:1.0:1783520936.143060:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002462720 00000100:00100000:1.0:1783520936.143070:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002462720 00000100:00100000:0.0:1783520936.143071:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44821 00000100:00100000:0.0:1783520936.143090:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513bc0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+747:3627:x1870152993242240:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.143104:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2054 count 1 00010000:00010000:0.0:1783520936.143115:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026639 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x806:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.143119:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b4098e11/0xd5401cbabc021d99 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21b8 expref: 262 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.143123:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993241088:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.143140:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000020ef181c/0xd5401cbabc022993 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x806:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3083 expref: 262 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.143149:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02660f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x802:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.143189:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x806:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.143192:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x806:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.143196:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026639 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x806:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.143204:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36405 00010000:00010000:0.0:1783520936.143214:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000020ef181c/0xd5401cbabc022993 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x806:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3083 expref: 262 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.143225:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503370e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+262:2387:x1870153002462720:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.143233:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000020ef181c/0xd5401cbabc022993 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x806:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3083 expref: 262 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.143243:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.143243:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee5500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993242368:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.143258:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b4098e11/0xd5401cbabc021d99 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x7ff:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21b8 expref: 262 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.143275:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b4098e11/0xd5401cbabc021d99 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x7ff:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21b8 expref: 262 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.143279:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514219780 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.143311:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b4098e11/0xd5401cbabc021d99 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x7ff:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21b8 expref: 262 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.143334:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ff:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.143338:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x7ff:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.143345:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026608 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.143356:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026608 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x7ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.143453:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.143473:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026608 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.143486:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026608 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.143509:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026608 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x7ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.143517:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503370e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+261:2387:x1870153002462720:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 300us (511us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.143532:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026608 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.143538:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36405 00010000:00010000:2.0:1783520936.143568:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.143624:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.143630:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026608 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.143639:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.143643:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026608 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x7ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.143651:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026608 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x7ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.143665:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026608 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x7ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.143672:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026608 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x7ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.143678:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+748:3625:x1870152993240448:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4006us (4231us total) trans 4294976316 rc 0/0 00000100:00100000:2.0:1783520936.143692:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44814 00010000:00010000:3.0:1783520936.143704:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.143710:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520936.143713:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e0f4e00/2, rc 0, transno 4294976316 00010000:00010000:3.0:1783520936.143715:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.143742:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e0f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993240448:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.143804:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350d164000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993241216:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.143830:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026616 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x801:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.143846:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350544c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993242496:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.143981:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.144017:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993242496 00000100:00100000:2.0:1783520936.144026:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993242496 00000100:00100000:0.0:1783520936.144062:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993242112:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.144081:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026632 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x805:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.144082:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44822 00000100:00100000:1.0:1783520936.144088:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350d165880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993241344:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.144102:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.144107:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350544e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+748:3625:x1870152993242496:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.144134:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2055 count 1 00010000:00010000:1.0:1783520936.144139:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02661d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x800:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.144146:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002462976 00010000:00010000:2.0:1783520936.144151:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026640 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x807:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.144158:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993241728:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.144160:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002462976 00010000:00010000:2.0:1783520936.144179:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000089c00ab6/0xd5401cbabc021dd8 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x807:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2228 expref: 261 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.144183:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026624 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x803:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.144189:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000944e9d51/0xd5401cbabc021da7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x801:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21d4 expref: 261 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.144192:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x807:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.144202:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x807:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.144212:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026640 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x807:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.144233:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36406 00010000:00010000:2.0:1783520936.144241:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000089c00ab6/0xd5401cbabc021dd8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x807:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2228 expref: 261 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.144248:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943629897c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+261:2387:x1870153002462976:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.144262:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000089c00ab6/0xd5401cbabc021dd8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x807:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2228 expref: 261 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.144264:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.144280:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993242624:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.144326:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607eb8300 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.144345:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000944e9d51/0xd5401cbabc021da7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x801:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21d4 expref: 261 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.144353:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc0380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993241984:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.144392:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02662b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x804:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.144434:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000944e9d51/0xd5401cbabc021da7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x801:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21d4 expref: 261 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.144450:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000944e9d51/0xd5401cbabc021da7 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x801:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21d4 expref: 261 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.144462:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x801:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.144466:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x801:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.144470:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026616 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x801:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.144478:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026616 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x801:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.144529:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.144540:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026616 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x801:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.144550:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026616 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x801:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.144554:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943629897c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+260:2387:x1870153002462976:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 311us (464us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.144571:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026616 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x801:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.144572:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36406 00010000:00010000:2.0:1783520936.144584:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026616 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x801:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.144603:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.144627:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.144635:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026616 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x801:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.144644:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.144652:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbbe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.144657:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026616 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x801:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.144665:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026616 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x801:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.144678:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026616 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x801:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.144686:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026616 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x801:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.144692:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c553100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+749:3628:x1870152993240832:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4152us (4341us total) trans 4294976317 rc 0/0 00000100:00100000:2.0:1783520936.144712:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44817 00000100:00100000:3.0:1783520936.144717:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.144741:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.144746:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520936.144747:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513bc2a00/2, rc 0, transno 4294976317 00010000:00010000:0.0:1783520936.144750:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.144767:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993240832:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.144769:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002463232 00000100:00100000:0.0:1783520936.144778:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002463232 00010000:00010000:0.0:1783520936.144805:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007ecb899c/0xd5401cbabc021db5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x803:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21f0 expref: 260 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.144807:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.144851:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350544df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993242752:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.144855:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002462848 00000100:00100000:0.0:1783520936.144862:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002462848 00010000:00010000:0.0:1783520936.144890:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c2e859c2/0xd5401cbabc021dae lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x802:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21e2 expref: 261 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.144901:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.144913:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.144953:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002463104 00000100:00100000:2.0:1783520936.144955:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002463488 00000100:00100000:0.0:1783520936.144958:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002463104 00000100:00100000:2.0:1783520936.144963:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002463488 00000100:00100000:3.0:1783520936.144965:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.144979:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000710fd570/0xd5401cbabc021da0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x800:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21c6 expref: 263 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.144992:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000004ca5903/0xd5401cbabc021dbc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x804:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21fe expref: 263 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.145009:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002463360 00000100:00100000:3.0:1783520936.145015:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002463360 00000100:00100000:0.0:1783520936.145019:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36407 00000100:00100000:1.0:1783520936.145034:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993242752 00000100:00100000:0.0:1783520936.145034:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503370380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+264:2387:x1870153002463232:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.145036:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36408 00010000:00010000:3.0:1783520936.145044:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000061799a66/0xd5401cbabc021dca lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x805:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f220c expref: 264 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.145046:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993242752 00010000:00010000:0.0:1783520936.145055:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.145058:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+264:2387:x1870153002462848:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.145066:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007ecb899c/0xd5401cbabc021db5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x803:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21f0 expref: 264 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.145073:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.145081:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c2e859c2/0xd5401cbabc021dae lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x802:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21e2 expref: 264 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.145084:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007ecb899c/0xd5401cbabc021db5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x803:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21f0 expref: 264 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.145086:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36409 00000100:00100000:1.0:1783520936.145099:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44823 00000100:00100000:3.0:1783520936.145108:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503372a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+264:2387:x1870153002463104:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.145123:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350544d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+749:3626:x1870152993242752:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.145131:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.145149:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007ecb899c/0xd5401cbabc021db5 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x803:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21f0 expref: 264 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.145152:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000710fd570/0xd5401cbabc021da0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x800:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21c6 expref: 264 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.145160:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c2e859c2/0xd5401cbabc021dae lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x802:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21e2 expref: 264 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.145164:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2056 count 1 00010000:00010000:3.0:1783520936.145185:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000710fd570/0xd5401cbabc021da0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x800:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21c6 expref: 264 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.145199:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026647 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x808:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.145200:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x803:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.145210:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c2e859c2/0xd5401cbabc021dae lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x802:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21e2 expref: 263 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.145213:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000710fd570/0xd5401cbabc021da0 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x800:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21c6 expref: 263 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.145222:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x803:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.145251:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026624 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x803:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.145275:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000bd37257c/0xd5401cbabc021de6 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x808:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2236 expref: 263 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.145283:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x802:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.145296:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026624 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x803:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.145311:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x802:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.145332:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02660f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x802:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.145340:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x800:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.145397:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02660f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x802:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.145418:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x808:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.145425:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x808:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.145437:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.145442:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x800:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.145448:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026647 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x808:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.145455:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02661d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x800:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.145479:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.145500:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+261:2387:x1870153002462848:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 422us (769us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.145522:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02661d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x800:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.145522:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503370380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+261:2387:x1870153002463232:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 488us (877us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.145528:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36408 00010000:00010000:1.0:1783520936.145546:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000bd37257c/0xd5401cbabc021de6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x808:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2236 expref: 261 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.145547:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36407 00010000:00010000:3.0:1783520936.145559:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.145569:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36410 00010000:00010000:1.0:1783520936.145590:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000bd37257c/0xd5401cbabc021de6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x808:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2236 expref: 259 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.145593:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36411 00000100:00100000:3.0:1783520936.145596:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503372a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+259:2387:x1870153002463104:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 490us (782us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.145596:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+259:2390:x1870153002463488:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.145622:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503370e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+259:2387:x1870153002463360:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.145635:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.145647:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36409 00010000:00010000:0.0:1783520936.145647:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.145656:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000004ca5903/0xd5401cbabc021dbc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x804:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21fe expref: 259 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.145661:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350544d500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993242880:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.145663:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000061799a66/0xd5401cbabc021dca lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x805:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f220c expref: 258 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.145702:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000061799a66/0xd5401cbabc021dca lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x805:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f220c expref: 258 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.145709:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000004ca5903/0xd5401cbabc021dbc lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x804:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21fe expref: 258 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.145726:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000061799a66/0xd5401cbabc021dca lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x805:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f220c expref: 258 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.145739:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000004ca5903/0xd5401cbabc021dbc lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x804:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f21fe expref: 257 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.145746:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x805:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.145753:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x805:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.145760:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026632 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x805:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.145761:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x804:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.145766:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470240 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.145769:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x804:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.145783:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02662b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x804:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.145785:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.145787:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026632 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x805:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.145803:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02662b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x804:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.145807:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02661d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x800:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.145846:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02661d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x800:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.145851:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.145858:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.145862:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02661d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x800:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.145874:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02661d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x800:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.145881:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+256:2390:x1870153002463488:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 294us (982us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.145895:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503370e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+256:2387:x1870153002463360:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 277us (930us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.145920:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36410 00000100:00100000:0.0:1783520936.145936:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36411 00000100:00100000:2.0:1783520936.145947:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002463616 00000100:00100000:2.0:1783520936.145954:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002463616 00010000:00010000:1.0:1783520936.145954:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520936.145970:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.145977:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000020ef181c/0xd5401cbabc022993 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x806:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3083 expref: 255 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.145993:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520936.145994:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee5500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993242368:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.145997:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002463744 00010000:00010000:3.0:1783520936.146000:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02661d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x800:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.146001:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002463744 00010000:00010000:2.0:1783520936.146014:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000089c00ab6/0xd5401cbabc021dd8 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x807:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2228 expref: 256 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.146016:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026639 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x806:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.146042:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36412 00000100:00100000:0.0:1783520936.146051:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36413 00000100:00100000:2.0:1783520936.146056:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+256:2390:x1870153002463616:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.146068:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.146068:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638584c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+256:2388:x1870153002463744:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.146075:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000020ef181c/0xd5401cbabc022993 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x806:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3083 expref: 256 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.146082:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.146086:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000020ef181c/0xd5401cbabc022993 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x806:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3083 expref: 256 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.146095:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000089c00ab6/0xd5401cbabc021dd8 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x807:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2228 expref: 256 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.146103:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000020ef181c/0xd5401cbabc022993 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x806:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3083 expref: 256 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.146112:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+750:3626:x1870152993240576:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6067us (6190us total) trans 4294976318 rc 0/0 00010000:00010000:0.0:1783520936.146114:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000089c00ab6/0xd5401cbabc021dd8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x807:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2228 expref: 256 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.146141:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.146144:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x806:0x0].0x0 since=0s v=(1 0) 00000100:00100000:1.0:1783520936.146147:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44815 00010000:00010000:3.0:1783520936.146154:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02661d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x800:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.146160:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x806:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.146167:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x807:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.146172:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026639 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x806:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.146172:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x807:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.146174:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02661d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x800:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.146181:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026640 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x807:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.146192:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c551500/2, rc 0, transno 4294976318 00010000:00010000:2.0:1783520936.146200:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026639 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x806:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.146209:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02661d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x800:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.146211:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026640 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x807:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.146212:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c551500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993240576:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.146245:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc02661d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x800:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.146251:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.146252:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.146269:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.146275:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.146279:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638584c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+255:2388:x1870153002463744:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 218us (321us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.146280:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.146289:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36413 00000100:00100000:2.0:1783520936.146290:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+255:2390:x1870153002463616:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 237us (506us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.146312:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36412 00010000:00010000:0.0:1783520936.146318:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026639 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x806:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.146320:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350544ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993243008:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.146340:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026639 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x806:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.146347:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026639 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x806:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.146352:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026639 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x806:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.146405:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02660f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x802:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.146410:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.146416:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520936.146418:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.146430:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026639 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x806:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.146433:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02660f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x802:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.146460:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02660f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x802:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.146462:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.146468:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026639 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x806:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.146480:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02660f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x802:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.146481:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026639 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x806:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.146483:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993243008 00000100:00100000:1.0:1783520936.146506:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993243008 00010000:00010000:3.0:1783520936.146527:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026639 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x806:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.146533:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.146534:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026639 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x806:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.146564:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.146570:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02660f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x802:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.146571:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513bc0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+751:3627:x1870152993242240:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3480us (3568us total) trans 4294976319 rc 0/0 00010000:00010000:3.0:1783520936.146587:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520936.146590:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44821 00000004:00080000:1.0:1783520936.146593:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350544ea00/2, rc 0, transno 4294976319 00010000:00010000:3.0:1783520936.146594:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02660f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x802:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.146611:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02660f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x802:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.146616:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350544ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993242240:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.146620:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44824 00000100:00100000:0.0:1783520936.146644:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513139c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+751:3627:x1870152993243008:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.146649:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02660f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x802:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.146651:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c550000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+751:3628:x1870152993240960:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6322us (6415us total) trans 4294976320 rc 0/0 00010000:00010000:3.0:1783520936.146666:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02660f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x802:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.146670:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2057 count 1 00000100:00100000:2.0:1783520936.146671:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44816 00010000:00010000:0.0:1783520936.146698:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02664e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x809:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.146706:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:3.0:1783520936.146711:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.146715:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000046804f8c/0xd5401cbabc021ded lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x809:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2244 expref: 253 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.146716:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.146720:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351313a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993243136:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.146722:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.146735:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.146743:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x809:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.146748:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x809:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.146755:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02664e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x809:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.146760:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000089c00ab6/0xd5401cbabc021dd8 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x807:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2228 expref: 253 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.146778:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.146792:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993242624:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.146793:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000046804f8c/0xd5401cbabc021ded lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x809:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2244 expref: 252 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.146812:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000046804f8c/0xd5401cbabc021ded lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x809:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2244 expref: 252 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.146823:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026640 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x807:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.146830:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee5500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993243264:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.146840:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026640 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x807:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.146850:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026640 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x807:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.146857:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513bc0a80/2, rc 0, transno 4294976320 00010000:00010000:2.0:1783520936.146861:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026640 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x807:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.146867:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993240960:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.146882:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026640 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x807:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.146912:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.146912:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895840 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.146919:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513138700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993243392:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.146928:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.146939:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026640 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x807:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.146953:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.146962:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.146968:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026640 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x807:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.146986:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026640 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x807:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.147003:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026640 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x807:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.147012:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026640 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x807:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.147019:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993243136 00000100:00100000:1.0:1783520936.147025:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993243136 00010000:00010000:0.0:1783520936.147047:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.147052:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.147056:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.147058:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350544e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+752:3625:x1870152993242496:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2959us (3075us total) trans 4294976321 rc 0/0 00000004:00080000:1.0:1783520936.147066:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350544c700/2, rc 0, transno 4294976321 00000100:00100000:2.0:1783520936.147071:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44822 00000100:00100000:1.0:1783520936.147086:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350544c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993242496:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.147089:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026632 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x805:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.147103:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026632 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x805:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.147107:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993243392 00000100:00100000:2.0:1783520936.147114:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993243392 00010000:00010000:0.0:1783520936.147114:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026632 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x805:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.147126:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026632 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x805:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.147143:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.147149:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44825 00010000:00010000:3.0:1783520936.147150:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.147155:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026632 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x805:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.147161:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351313b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+752:3625:x1870152993243392:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.147168:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.147173:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026632 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x805:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.147182:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026632 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x805:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.147186:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2059 count 1 00010000:00010000:3.0:1783520936.147195:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026632 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x805:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.147202:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026632 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x805:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.147204:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026655 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x80b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.147206:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513139880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993243520:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.147237:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004b32f2d2/0xd5401cbabc021dfb lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x80b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2260 expref: 252 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.147239:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.147243:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.147248:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.147249:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x80b:0x0].0x0 since=38s v=(0 0) 00000100:00100000:0.0:1783520936.147253:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638997800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+753:3627:x1870152993241856:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4561us (4685us total) trans 4294976322 rc 0/0 00010000:00010000:2.0:1783520936.147255:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x80b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.147264:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026655 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x80b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.147273:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.147275:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44820 00010000:00010000:2.0:1783520936.147306:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004b32f2d2/0xd5401cbabc021dfb lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x80b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2260 expref: 252 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.147307:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993243520 00000100:00100000:0.0:1783520936.147313:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993243520 00010000:00010000:2.0:1783520936.147322:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004b32f2d2/0xd5401cbabc021dfb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x80b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2260 expref: 252 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.147338:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993243648:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.147348:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44826 00000004:00080000:1.0:1783520936.147349:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943638995500/2, rc 0, transno 4294976322 00000100:00100000:1.0:1783520936.147386:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943638995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993241856:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.147390:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351313a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+753:3626:x1870152993243520:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.147410:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2060 count 1 00010000:00010000:0.0:1783520936.147427:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02665c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x80c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.147444:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a1c06498/0xd5401cbabc021e09 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x80c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f226e expref: 252 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.147453:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x80c:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.147458:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x80c:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.147462:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513138380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993243776:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.147465:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57d80 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.147466:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02665c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x80c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.147503:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.147511:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a1c06498/0xd5401cbabc021e09 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x80c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f226e expref: 252 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.147520:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02662b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x804:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.147529:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a1c06498/0xd5401cbabc021e09 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x80c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f226e expref: 252 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.147532:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02662b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x804:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.147555:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02662b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x804:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.147559:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993243904:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.147569:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993243776 00010000:00010000:2.0:1783520936.147570:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02662b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x804:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.147578:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993243776 00010000:00010000:2.0:1783520936.147592:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.147613:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895600 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.147622:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44827 00000100:00100000:1.0:1783520936.147636:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351313b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+755:3627:x1870152993243136:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.147638:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026624 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x803:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.147645:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.147655:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02662b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x804:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.147656:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026624 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x803:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.147660:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2058 count 1 00010000:00010000:3.0:1783520936.147683:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.147683:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026624 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x803:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.147686:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026663 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x80a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.147691:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02662b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x804:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.147707:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638995180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+755:3625:x1870152993241600:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5217us (5429us total) trans 4294976323 rc 0/0 00000100:00100000:2.0:1783520936.147746:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44819 00010000:00010000:0.0:1783520936.147746:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026624 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x803:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.147749:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02662b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x804:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.147750:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000021374616/0xd5401cbabc021df4 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x80a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2252 expref: 252 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.147780:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44828 00000100:00100000:2.0:1783520936.147804:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513138000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+754:3627:x1870152993243776:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.147804:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x80a:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.147812:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.147816:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x80a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.147817:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02662b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x804:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.147827:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026663 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x80a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.147835:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2061 count 1 00010000:00010000:3.0:1783520936.147856:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02662b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x804:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.147861:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02666a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x80d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.147882:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008a08d338/0xd5401cbabc021e10 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x80d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f227c expref: 252 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.147888:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c551c00/2, rc 0, transno 4294976323 00010000:00010000:2.0:1783520936.147889:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x80d:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.147890:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.147901:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026624 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x803:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.147901:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x80d:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.147910:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c551c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993241600:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.147915:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02666a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x80d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.147915:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350d166a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+755:3625:x1870152993241472:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5749us (5891us total) trans 4294976324 rc 0/0 00000004:00080000:1.0:1783520936.147936:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350d165f80/2, rc 0, transno 4294976324 00010000:00010000:3.0:1783520936.147943:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbebe00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520936.147952:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44818 00000100:00100000:1.0:1783520936.147954:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350d165f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993241472:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.147957:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026624 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x803:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.147974:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008a08d338/0xd5401cbabc021e10 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x80d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f227c expref: 252 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.148008:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026624 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x803:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.148009:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008a08d338/0xd5401cbabc021e10 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x80d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f227c expref: 252 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.148021:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1783520936.148041:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363fe54700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993244288:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.148043:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026624 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x803:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.148047:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc0380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993244032:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.148065:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026624 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x803:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.148086:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.148088:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.148091:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.148102:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607eb8300 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.148103:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.148111:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.148144:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993244288 00000100:00100000:2.0:1783520936.148153:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993244288 00000100:00100000:1.0:1783520936.148160:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1783520936.148169:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513139500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993244160:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.148198:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.148206:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993244160 00000100:00100000:2.0:1783520936.148208:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44829 00000100:00100000:0.0:1783520936.148212:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993244160 00000100:00100000:2.0:1783520936.148224:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363fe54a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+756:3626:x1870152993244288:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.148239:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2063 count 1 00000100:00100000:0.0:1783520936.148247:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44830 00010000:00010000:2.0:1783520936.148252:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026671 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x80f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.148259:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363fe56a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+756:3627:x1870152993244160:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.148268:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000021374616/0xd5401cbabc021df4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x80a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2252 expref: 252 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.148277:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000507b550a/0xd5401cbabc021e25 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x80f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2298 expref: 252 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.148283:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2062 count 1 00010000:00010000:1.0:1783520936.148296:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000021374616/0xd5401cbabc021df4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x80a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2252 expref: 252 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.148298:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x80f:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.148302:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026678 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x80e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.148315:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x80f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.148335:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026671 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x80f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.148355:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363fe55880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993244416:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.148380:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003c315ade/0xd5401cbabc022882 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x80e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f72 expref: 252 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.148395:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x80e:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.148399:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x80e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.148404:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026678 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x80e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.148422:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000507b550a/0xd5401cbabc021e25 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x80f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2298 expref: 252 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.148440:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000507b550a/0xd5401cbabc021e25 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x80f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2298 expref: 252 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.148449:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003c315ade/0xd5401cbabc022882 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x80e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f72 expref: 252 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.148458:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993244544:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.148466:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003c315ade/0xd5401cbabc022882 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x80e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f72 expref: 252 pid: 8408 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.148480:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993244672:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.148540:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470840 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.148541:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514219780 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.148542:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514825600 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.148574:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.148598:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee5500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993243264:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.148606:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350544d500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993242880:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.148618:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993243648:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.148623:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02664e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x809:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.148645:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.148648:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026647 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x808:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.148650:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026655 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x80b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.148659:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002464000 00000100:00100000:0.0:1783520936.148665:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002464000 00000100:00100000:3.0:1783520936.148667:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002463872 00000100:00100000:3.0:1783520936.148675:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002463872 00010000:00010000:0.0:1783520936.148688:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000bd37257c/0xd5401cbabc021de6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x808:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2236 expref: 254 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.148700:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000046804f8c/0xd5401cbabc021ded lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x809:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2244 expref: 254 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.148731:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36414 00000100:00100000:0.0:1783520936.148739:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+254:2387:x1870153002464000:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.148749:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.148754:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000bd37257c/0xd5401cbabc021de6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x808:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2236 expref: 254 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.148767:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000bd37257c/0xd5401cbabc021de6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x808:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2236 expref: 254 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.148782:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000bd37257c/0xd5401cbabc021de6 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x808:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2236 expref: 254 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.148795:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x808:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.148798:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x808:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.148804:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026647 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x808:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.148813:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36415 00010000:00010000:0.0:1783520936.148813:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026647 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x808:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.148825:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+253:2387:x1870153002463872:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.148835:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.148841:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000046804f8c/0xd5401cbabc021ded lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x809:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2244 expref: 253 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.148841:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.148861:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000046804f8c/0xd5401cbabc021ded lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x809:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2244 expref: 253 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.148868:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+253:2387:x1870153002464000:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 127us (298us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.148872:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026647 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x808:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.148885:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000046804f8c/0xd5401cbabc021ded lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x809:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2244 expref: 253 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.148885:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36414 00010000:00010000:1.0:1783520936.148906:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026647 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x808:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.148915:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x809:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.148921:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x809:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.148921:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026647 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x808:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.148939:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02664e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x809:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.148944:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026647 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x808:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.148982:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.148996:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.149003:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026647 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x808:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.149018:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02664e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x809:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.149018:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.149025:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026647 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x808:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.149039:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026647 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x808:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.149047:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.149051:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026647 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x808:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.149060:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026647 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x808:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.149080:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.149082:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.149085:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.149098:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02664e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x809:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.149106:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02664e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x809:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.149117:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350544d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+757:3626:x1870152993242752:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4002us (4210us total) trans 4294976325 rc 0/0 00010000:00010000:0.0:1783520936.149117:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02664e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x809:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.149144:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44823 00000100:00100000:3.0:1783520936.149159:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+251:2387:x1870153002463872:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 332us (514us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.149165:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02664e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x809:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.149175:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36415 00010000:00010000:0.0:1783520936.149189:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000004:00080000:1.0:1783520936.149206:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350544df80/2, rc 0, transno 4294976325 00010000:00010000:3.0:1783520936.149208:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.149216:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02664e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x809:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.149221:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350544df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993242752:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.149241:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.149247:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02664e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x809:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.149262:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02664e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x809:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.149276:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02664e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x809:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.149285:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc02664e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x809:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.149304:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513139c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+757:3627:x1870152993243008:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2665us (2892us total) trans 4294976326 rc 0/0 00010000:00010000:3.0:1783520936.149310:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.149316:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363fe54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993244800:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.149317:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.149317:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44824 00010000:00010000:3.0:1783520936.149324:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.149420:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.149438:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000004:00080000:0.0:1783520936.149450:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350544ed80/2, rc 0, transno 4294976326 00000100:00100000:0.0:1783520936.149475:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350544ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993243008:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.149541:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993243904:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.149575:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02665c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x80c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.149580:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363fe57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993244928:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.149617:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993244800 00000100:00100000:0.0:1783520936.149621:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993244800 00000100:00100000:1.0:1783520936.149629:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.149670:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993244928 00000100:00100000:2.0:1783520936.149678:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44831 00000100:00100000:0.0:1783520936.149678:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993244928 00000100:00100000:2.0:1783520936.149694:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363fe57800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+758:3626:x1870152993244800:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.149700:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002464128 00000100:00100000:3.0:1783520936.149706:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002464128 00002000:00080000:2.0:1783520936.149712:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2064 count 1 00000100:00100000:0.0:1783520936.149713:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44832 00000100:00100000:0.0:1783520936.149726:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363fe54000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+758:3627:x1870152993244928:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.149728:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004b32f2d2/0xd5401cbabc021dfb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x80b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2260 expref: 251 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.149731:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02667f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x810:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.149745:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2065 count 1 00010000:00010000:2.0:1783520936.149759:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000022e6ed46/0xd5401cbabc021e2c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x810:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f22a6 expref: 251 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.149763:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026686 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x811:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.149769:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x810:0x0].0x0 since=38s v=(0 0) 00000100:00100000:3.0:1783520936.149775:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36416 00010000:00010000:2.0:1783520936.149777:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x810:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.149786:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02667f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x810:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.149789:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000647f27c7/0xd5401cbabc021e33 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x811:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f22b4 expref: 251 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.149791:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503372300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+251:2390:x1870153002464128:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.149806:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.149814:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004b32f2d2/0xd5401cbabc021dfb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x80b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2260 expref: 251 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.149820:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x811:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.149828:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000022e6ed46/0xd5401cbabc021e2c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x810:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22a6 expref: 251 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.149830:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x811:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.149835:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004b32f2d2/0xd5401cbabc021dfb lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x80b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2260 expref: 251 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.149840:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026686 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x811:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.149855:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000022e6ed46/0xd5401cbabc021e2c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x810:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22a6 expref: 251 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.149870:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004b32f2d2/0xd5401cbabc021dfb lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x80b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2260 expref: 251 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.149876:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993245056:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.149883:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000647f27c7/0xd5401cbabc021e33 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x811:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22b4 expref: 251 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.149887:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x80b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.149896:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x80b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.149902:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000647f27c7/0xd5401cbabc021e33 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x811:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22b4 expref: 250 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.149903:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026655 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.149926:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026655 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x80b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.149928:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57d80 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.149931:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993245184:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.149959:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.149965:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895600 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.149974:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026655 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.149987:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503372300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+250:2390:x1870153002464128:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 201us (550us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.150000:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026655 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.150004:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36416 00010000:00010000:2.0:1783520936.150016:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026655 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.150024:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026655 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x80b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.150049:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520936.150051:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.150073:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.150079:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026655 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x80b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.150079:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002464256 00000100:00100000:0.0:1783520936.150085:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002464256 00010000:00010000:3.0:1783520936.150090:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.150095:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026655 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x80b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.150104:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026655 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x80b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.150108:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a1c06498/0xd5401cbabc021e09 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x80c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f226e expref: 250 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.150124:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026655 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x80b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.150134:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026655 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x80b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.150141:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36417 00000100:00100000:2.0:1783520936.150152:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351313b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+759:3625:x1870152993243392:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2994us (3202us total) trans 4294976327 rc 0/0 00000100:00100000:0.0:1783520936.150153:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+250:2387:x1870153002464256:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.150167:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44825 00010000:00010000:0.0:1783520936.150168:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.150175:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a1c06498/0xd5401cbabc021e09 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x80c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f226e expref: 250 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.150196:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513138700/2, rc 0, transno 4294976327 00010000:00010000:0.0:1783520936.150197:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a1c06498/0xd5401cbabc021e09 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x80c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f226e expref: 250 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.150210:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513138700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993243392:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.150226:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a1c06498/0xd5401cbabc021e09 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x80c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f226e expref: 250 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.150228:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.150232:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.150236:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.150246:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x80c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.150251:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x80c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.150259:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02665c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.150268:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02665c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x80c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.150287:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:0.0:1783520936.150295:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.150302:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363fe56300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993245312:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.150328:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+249:2387:x1870153002464256:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 176us (278us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.150337:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc0380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993244032:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.150343:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36417 00000100:00100000:1.0:1783520936.150382:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.150395:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02666a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x80d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.150427:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02665c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.150437:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02665c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.150441:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.150454:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993245312 00010000:00010000:0.0:1783520936.150454:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02665c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.150463:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993245312 00010000:00010000:0.0:1783520936.150468:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02665c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x80c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.150503:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002464384 00000100:00100000:0.0:1783520936.150508:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002464384 00000100:00100000:1.0:1783520936.150533:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44833 00010000:00010000:0.0:1783520936.150536:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008a08d338/0xd5401cbabc021e10 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x80d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f227c expref: 249 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.150552:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363fe55180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+759:3627:x1870152993245312:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.150571:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36418 00002000:00080000:1.0:1783520936.150575:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2066 count 1 00000100:00100000:0.0:1783520936.150586:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503370380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+249:2388:x1870153002464384:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520936.150587:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.150602:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02668d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x812:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.150603:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.150614:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993244544:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.150618:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008a08d338/0xd5401cbabc021e10 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x80d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f227c expref: 249 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.150642:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026671 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x80f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.150649:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002464512 00000100:00100000:3.0:1783520936.150657:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002464512 00010000:00010000:1.0:1783520936.150666:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008e3141e2/0xd5401cbabc021e41 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x812:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f22c2 expref: 250 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.150669:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008a08d338/0xd5401cbabc021e10 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x80d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f227c expref: 250 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.150677:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000021374616/0xd5401cbabc021df4 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x80a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2252 expref: 250 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.150702:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x812:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.150723:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x812:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.150731:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02668d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x812:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.150738:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008a08d338/0xd5401cbabc021e10 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x80d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f227c expref: 250 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.150740:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36419 00000100:00100000:3.0:1783520936.150756:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503372a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+250:2388:x1870153002464512:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.150773:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.150783:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000021374616/0xd5401cbabc021df4 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x80a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2252 expref: 250 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.150789:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x80d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.150793:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000021374616/0xd5401cbabc021df4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x80a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2252 expref: 249 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.150796:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x80d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.150809:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008e3141e2/0xd5401cbabc021e41 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x812:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22c2 expref: 249 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.150811:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02666a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.150816:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x80a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.150833:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x80a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.150847:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026663 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.150862:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008e3141e2/0xd5401cbabc021e41 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x812:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22c2 expref: 249 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.150865:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026663 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x80a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.150889:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02666a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x80d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.150920:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.150925:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363fe56d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993245440:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.150940:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.150956:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503372a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+249:2388:x1870153002464512:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 200us (369us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.150962:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02666a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.150969:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36419 00000100:00100000:1.0:1783520936.150980:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470780 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.150983:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503370380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+249:2388:x1870153002464384:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 397us (631us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.150997:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02666a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.150999:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36418 00010000:00010000:2.0:1783520936.151012:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02666a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.151020:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02666a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x80d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.151031:0:9151:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000021374616/0xd5401cbabc021df4 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x80a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2252 expref: 247 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.151040:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:0.0:1783520936.151047:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:1.0:1783520936.151053:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363fe55880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993244416:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.151062:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520936.151070:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02666a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x80d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.151072:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.151076:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026663 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.151081:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02665c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x80c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.151111:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.151119:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026663 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.151123:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02666a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x80d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.151128:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.151142:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02665c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x80c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.151148:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513138000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+760:3627:x1870152993243776:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3344us (3647us total) trans 4294976328 rc 0/0 00010000:00010000:1.0:1783520936.151148:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026663 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.151186:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02666a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x80d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.151188:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44828 00010000:00010000:1.0:1783520936.151189:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026663 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.151228:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02666a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x80d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.151238:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026663 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x80a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.151243:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02665c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x80c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.151256:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc02666a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x80d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.151318:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513138380/2, rc 0, transno 4294976328 00010000:00010000:0.0:1783520936.151321:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.151325:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.151328:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.151332:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513138380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993243776:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.151352:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02665c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x80c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.151462:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363fe55880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993245568:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.151473:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351313a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+760:3626:x1870152993243520:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4085us (4202us total) trans 4294976329 rc 0/0 00000100:00100000:0.0:1783520936.151502:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44826 00010000:00010000:3.0:1783520936.151508:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc02665c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x80c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.151523:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.151529:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.151533:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.151539:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.151573:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993245568 00000100:00100000:2.0:1783520936.151581:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993245568 00000100:00100000:3.0:1783520936.151582:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000004:00080000:1.0:1783520936.151604:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513139880/2, rc 0, transno 4294976329 00000100:00100000:1.0:1783520936.151619:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513139880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993243520:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.151627:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44834 00000100:00100000:3.0:1783520936.151649:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.151649:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363fe55c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+760:3625:x1870152993245568:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.151659:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993244672:192.168.204.33@tcp:104: 00002000:00080000:2.0:1783520936.151672:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2067 count 1 00010000:00010000:0.0:1783520936.151684:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026678 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x80e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.151695:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026694 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x813:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.151703:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350544d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993245696:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.151729:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000011451c31/0xd5401cbabc020fbc lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x813:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f22d0 expref: 246 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.151739:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x813:0x0].0x0 since=38s v=(0 0) 00000100:00100000:0.0:1783520936.151740:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993245184:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.151748:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x813:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.151750:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.151758:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026694 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x813:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.151764:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026686 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x811:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.151776:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.151786:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000011451c31/0xd5401cbabc020fbc lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x813:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22d0 expref: 246 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.151802:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000011451c31/0xd5401cbabc020fbc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x813:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22d0 expref: 246 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.151804:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002464640 00000100:00100000:0.0:1783520936.151810:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002464640 00000100:00100000:2.0:1783520936.151825:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993245824:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.151836:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000507b550a/0xd5401cbabc021e25 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x80f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2298 expref: 247 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.151850:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.151863:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.151872:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002464768 00000100:00100000:0.0:1783520936.151876:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002464768 00010000:00010000:3.0:1783520936.151886:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:2.0:1783520936.151890:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a87a80 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.151891:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003c315ade/0xd5401cbabc022882 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x80e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f72 expref: 248 pid: 8408 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.151893:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026663 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x80a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.151910:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993245696 00010000:00010000:3.0:1783520936.151916:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbba00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520936.151917:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993245696 00000100:00100000:0.0:1783520936.151926:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002465024 00000100:00100000:0.0:1783520936.151931:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002465024 00010000:00010000:0.0:1783520936.151949:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000647f27c7/0xd5401cbabc021e33 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x811:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22b4 expref: 249 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.151957:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44835 00000100:00100000:2.0:1783520936.151971:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350793a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+762:3626:x1870152993245696:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.151972:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002464896 00000100:00100000:0.0:1783520936.151977:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002464896 00002000:00080000:2.0:1783520936.151988:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2068 count 1 00010000:00010000:0.0:1783520936.151997:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000022e6ed46/0xd5401cbabc021e2c lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x810:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22a6 expref: 250 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.152003:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc02669b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x814:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.152014:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351313b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+762:3627:x1870152993243136:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4376us (5237us total) trans 4294976330 rc 0/0 00010000:00010000:3.0:1783520936.152021:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026663 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x80a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.152022:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36420 00010000:00010000:2.0:1783520936.152034:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000030e97e7b/0xd5401cbabc020fdf lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x814:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f22de expref: 250 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.152042:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943629897450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+250:2388:x1870153002464640:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.152057:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44827 00010000:00010000:0.0:1783520936.152061:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.152073:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x814:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.152075:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000507b550a/0xd5401cbabc021e25 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x80f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2298 expref: 250 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.152079:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x814:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.152088:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc02669b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x814:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.152099:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000507b550a/0xd5401cbabc021e25 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x80f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2298 expref: 250 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.152115:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351313a300/2, rc 0, transno 4294976330 00010000:00010000:3.0:1783520936.152121:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026663 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x80a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.152121:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000030e97e7b/0xd5401cbabc020fdf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x814:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22de expref: 250 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.152123:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000507b550a/0xd5401cbabc021e25 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x80f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2298 expref: 250 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.152133:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351313a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993243136:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.152170:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026663 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x80a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.152179:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x80f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.152180:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000030e97e7b/0xd5401cbabc020fdf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x814:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22de expref: 249 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.152183:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026663 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x80a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.152184:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x80f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.152197:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026671 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.152210:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1783520936.152214:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc0380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993245952:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.152230:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350793b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993246080:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.152231:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026671 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x80f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.152244:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.152253:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.152256:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514825600 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.152259:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.152275:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.152284:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026671 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.152297:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026671 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.152305:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993245056:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.152307:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.152310:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943629897450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+249:2388:x1870153002464640:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 272us (725us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.152332:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026671 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.152334:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36420 00010000:00010000:3.0:1783520936.152341:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02667f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x810:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.152345:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026671 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x80f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.152433:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.152438:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993246080 00000100:00100000:1.0:1783520936.152444:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993246080 00000100:00100000:3.0:1783520936.152445:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36421 00010000:00010000:3.0:1783520936.152468:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.152475:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026671 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x80f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.152484:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184c00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520936.152486:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44836 00010000:00010000:3.0:1783520936.152505:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026671 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x80f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.152508:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36422 00000100:00100000:1.0:1783520936.152514:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507939c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+763:3628:x1870152993246080:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.152533:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503370e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+248:2388:x1870153002465024:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.152538:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026671 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x80f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.152542:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2069 count 1 00010000:00010000:0.0:1783520936.152547:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.152560:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000647f27c7/0xd5401cbabc021e33 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x811:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22b4 expref: 248 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.152567:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0266a2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x815:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.152568:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026671 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x80f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.152575:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000647f27c7/0xd5401cbabc021e33 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x811:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22b4 expref: 248 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.152578:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363fe54a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+763:3626:x1870152993244288:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4354us (4467us total) trans 4294976331 rc 0/0 00000100:00100000:2.0:1783520936.152616:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44829 00010000:00010000:3.0:1783520936.152626:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026671 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x80f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.152630:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000031bd9081/0xd5401cbabc02102c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x815:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f22ec expref: 248 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.152639:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000647f27c7/0xd5401cbabc021e33 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x811:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22b4 expref: 248 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.152666:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363fe54700/2, rc 0, transno 4294976331 00010000:00010000:1.0:1783520936.152681:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x815:0x0].0x0 since=38s v=(0 0) 00000100:00100000:2.0:1783520936.152691:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363fe54700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993244288:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.152694:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x815:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.152696:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x811:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.152699:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.152705:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.152710:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0266a2 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x815:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.152711:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.152711:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x811:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.152731:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026686 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x811:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.152735:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+247:2388:x1870153002464768:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.152758:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026686 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x811:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.152759:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.152768:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003c315ade/0xd5401cbabc022882 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x80e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f72 expref: 247 pid: 8408 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.152786:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993246208:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.152793:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003c315ade/0xd5401cbabc022882 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x80e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f72 expref: 247 pid: 8408 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.152797:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.152813:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003c315ade/0xd5401cbabc022882 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x80e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f72 expref: 247 pid: 8408 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.152825:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x80e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.152831:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x80e:0x0].0x0 since=0s v=(1 0) 00000100:00100000:0.0:1783520936.152831:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503370e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+246:2388:x1870153002465024:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 306us (1084us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.152839:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026678 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.152853:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.152854:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36422 00010000:00010000:3.0:1783520936.152866:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026678 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x80e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.152890:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36423 00010000:00010000:3.0:1783520936.152896:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.152908:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+245:2388:x1870153002464896:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.152910:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.152925:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.152933:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000022e6ed46/0xd5401cbabc021e2c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x810:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22a6 expref: 245 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.152938:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000031bd9081/0xd5401cbabc02102c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x815:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22ec expref: 245 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.152948:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993246208 00000100:00100000:2.0:1783520936.152959:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993246208 00010000:00010000:1.0:1783520936.152977:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000031bd9081/0xd5401cbabc02102c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x815:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22ec expref: 245 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.152984:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+245:2388:x1870153002464768:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 479us (1292us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.152987:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000022e6ed46/0xd5401cbabc021e2c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x810:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22a6 expref: 245 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.153001:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44837 00000100:00100000:2.0:1783520936.153020:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507939880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+763:3628:x1870152993246208:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.153033:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507938a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993246336:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.153036:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36421 00010000:00010000:0.0:1783520936.153045:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000022e6ed46/0xd5401cbabc021e2c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x810:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22a6 expref: 245 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.153048:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2070 count 1 00010000:00010000:0.0:1783520936.153064:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x810:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.153067:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002465152 00010000:00010000:0.0:1783520936.153068:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x810:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.153073:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0266a9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x816:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.153074:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002465152 00010000:00010000:0.0:1783520936.153074:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02667f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x810:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.153092:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124706c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.153096:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02667f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x810:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.153100:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008e3141e2/0xd5401cbabc021e41 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x812:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22c2 expref: 244 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.153103:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009713add6/0xd5401cbabc022adc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x816:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f31cc expref: 244 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.153131:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x816:0x0].0x0 since=37s v=(0 0) 00000100:00100000:1.0:1783520936.153137:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363fe56d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993245440:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.153140:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x816:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.153147:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.153151:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36424 00010000:00010000:2.0:1783520936.153153:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0266a9 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x816:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.153167:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513bc1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+244:2388:x1870153002465152:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.153171:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02668d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x812:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.153178:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.153188:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009713add6/0xd5401cbabc022adc lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x816:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31cc expref: 244 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.153188:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+244:2388:x1870153002464896:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 283us (1340us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.153191:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008e3141e2/0xd5401cbabc021e41 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x812:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22c2 expref: 244 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.153209:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36423 00010000:00010000:3.0:1783520936.153218:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008e3141e2/0xd5401cbabc021e41 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x812:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22c2 expref: 244 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.153221:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009713add6/0xd5401cbabc022adc lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x816:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31cc expref: 244 pid: 20140 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.153237:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008e3141e2/0xd5401cbabc021e41 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x812:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22c2 expref: 243 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.153242:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993246464:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.153250:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x812:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.153254:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x812:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.153257:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02668d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x812:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.153260:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026678 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.153264:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02668d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x812:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.153281:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026678 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.153283:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607eb8300 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.153295:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026678 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x80e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.153296:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.153306:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026678 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x80e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.153323:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513bc1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+242:2388:x1870153002465152:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 162us (425us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.153323:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02668d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x812:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.153326:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.153345:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:3.0:1783520936.153351:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36424 00010000:00010000:0.0:1783520936.153355:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026678 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x80e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.153405:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02668d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x812:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.153417:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02668d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x812:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.153425:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02668d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x812:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.153460:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.153466:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026678 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x80e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.153471:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02667f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x810:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.153477:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.153511:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02667f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x810:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.153514:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026678 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x80e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.153526:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02667f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x810:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.153532:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02667f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x810:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.153534:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026678 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x80e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.153548:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026678 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x80e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.153549:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.153557:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02668d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x812:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.153567:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.153572:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.153574:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02668d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x812:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.153576:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.153584:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.153586:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02668d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x812:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.153604:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02668d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x812:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.153613:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02668d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x812:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.153641:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363fe55180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+765:3627:x1870152993245312:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3095us (3278us total) trans 4294976332 rc 0/0 00000100:00100000:1.0:1783520936.153650:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44833 00010000:00010000:3.0:1783520936.153671:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.153675:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.153681:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.153700:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000004:00080000:1.0:1783520936.153704:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363fe56300/2, rc 0, transno 4294976332 00010000:00010000:3.0:1783520936.153709:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520936.153715:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363fe56a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+764:3627:x1870152993244160:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5453us (5511us total) trans 4294976333 rc 0/0 00010000:00010000:3.0:1783520936.153720:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02667f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x810:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.153728:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44830 00000100:00100000:1.0:1783520936.153736:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363fe56300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993245312:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.153738:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.153753:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02667f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x810:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.153766:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02667f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x810:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.153767:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026686 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x811:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.153772:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513139500/2, rc 0, transno 4294976333 00000100:00100000:1.0:1783520936.153790:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513139500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993244160:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.153792:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02667f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x810:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.153792:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026686 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x811:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.153818:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02667f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x810:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.153820:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026686 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x811:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.153846:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350d165880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993246592:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.153847:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026686 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x811:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.153872:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350d164000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993246720:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.153873:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.153875:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to regular blp list, 0 blwis in pool 00010000:00010000:3.0:1783520936.153881:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026686 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x811:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.153890:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.153894:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026686 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x811:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.153903:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026686 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x811:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.153906:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.153927:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026686 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x811:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.153928:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.153937:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026686 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x811:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.153962:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.153968:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.153973:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.153979:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993246592 00010000:00010000:3.0:1783520936.153981:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.153986:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993246592 00000100:00100000:0.0:1783520936.153992:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363fe54000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+764:3627:x1870152993244928:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4266us (4363us total) trans 4294976334 rc 0/0 00000100:00100000:0.0:1783520936.154001:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44832 00000100:00100000:0.0:1783520936.154021:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993246720 00000100:00100000:0.0:1783520936.154026:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993246720 00000004:00080000:1.0:1783520936.154030:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363fe57480/2, rc 0, transno 4294976334 00000100:00100000:1.0:1783520936.154050:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363fe57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993244928:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.154051:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44838 00000100:00100000:0.0:1783520936.154069:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032e9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+766:3626:x1870152993246720:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.154080:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2072 count 1 00010000:00010000:0.0:1783520936.154091:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0266b0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x818:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.154096:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363fe57800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+766:3626:x1870152993244800:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4405us (4675us total) trans 4294976335 rc 0/0 00010000:00010000:0.0:1783520936.154107:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000094b21e9e/0xd5401cbabc0210f7 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x818:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2316 expref: 241 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.154108:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44831 00010000:00010000:0.0:1783520936.154117:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x818:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.154120:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x818:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.154123:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0266b0 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x818:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.154131:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032eb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993246848:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.154139:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000094b21e9e/0xd5401cbabc0210f7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x818:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2316 expref: 241 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.154147:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000094b21e9e/0xd5401cbabc0210f7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x818:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2316 expref: 241 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.154156:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363fe54380/2, rc 0, transno 4294976335 00000100:00100000:0.0:1783520936.154162:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993246976:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.154174:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363fe54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993244800:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.154183:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.154216:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993246848 00000100:00100000:3.0:1783520936.154223:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993246848 00000100:00100000:0.0:1783520936.154224:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895840 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.154235:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1783520936.154249:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993247104:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.154256:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44839 00000100:00100000:0.0:1783520936.154267:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635739c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+766:3625:x1870152993246592:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.154278:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2071 count 1 00010000:00010000:0.0:1783520936.154289:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0266b7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x817:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.154302:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a5da0c5f/0xd5401cbabc0210aa lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x817:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2308 expref: 241 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.154309:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.154317:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x817:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.154320:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x817:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.154323:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0266b7 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x817:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.154338:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a5da0c5f/0xd5401cbabc0210aa lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x817:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2308 expref: 241 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.154351:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a5da0c5f/0xd5401cbabc0210aa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x817:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2308 expref: 241 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.154392:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993247232:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.154400:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993247104 00000100:00100000:1.0:1783520936.154408:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993247104 00000100:00100000:3.0:1783520936.154446:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44840 00000100:00100000:0.0:1783520936.154456:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514219780 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.154467:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44841 00000100:00100000:3.0:1783520936.154468:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350793bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+767:3627:x1870152993246848:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.154501:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507939180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+767:3627:x1870152993247104:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520936.154505:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2073 count 1 00010000:00010000:3.0:1783520936.154522:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0266be lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x819:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.154523:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2074 count 1 00010000:00010000:3.0:1783520936.154543:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002bbdf4b5/0xd5401cbabc02112f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x819:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2324 expref: 241 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.154547:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0266c5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x81a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.154565:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x819:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.154573:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x819:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.154578:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c9f0710e/0xd5401cbabc021167 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x81a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2332 expref: 241 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.154580:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0266be lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x819:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.154592:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x81a:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.154596:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x81a:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.154599:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc3480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993245824:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.154603:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0266c5 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x81a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.154620:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026694 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x813:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.154635:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c9f0710e/0xd5401cbabc021167 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x81a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2332 expref: 241 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.154650:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc0380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993245952:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.154685:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc02669b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x814:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.154687:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c9f0710e/0xd5401cbabc021167 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x81a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2332 expref: 241 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.154740:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507938380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993247360:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.154765:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002bbdf4b5/0xd5401cbabc02112f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x819:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2324 expref: 241 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.154775:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002bbdf4b5/0xd5401cbabc02112f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x819:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2324 expref: 241 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.154784:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503371f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993247488:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.154794:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470e40 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.154827:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6600 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.154843:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507938a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993246336:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.154855:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.154878:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0266a2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x815:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.154911:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002465408 00000100:00100000:1.0:1783520936.154915:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002465408 00010000:00010000:1.0:1783520936.154944:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000030e97e7b/0xd5401cbabc020fdf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x814:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22de expref: 242 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.154958:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.154999:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002465280 00000100:00100000:1.0:1783520936.155005:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002465280 00000100:00100000:3.0:1783520936.155014:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36425 00010000:00010000:1.0:1783520936.155028:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000011451c31/0xd5401cbabc020fbc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x813:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22d0 expref: 243 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.155039:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.155085:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002465536 00000100:00100000:1.0:1783520936.155096:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36426 00000100:00100000:0.0:1783520936.155097:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002465536 00000100:00100000:2.0:1783520936.155098:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993246464:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.155123:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032e8700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+244:2388:x1870153002465280:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.155126:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0266a9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x816:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.155130:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000031bd9081/0xd5401cbabc02102c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x815:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22ec expref: 244 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.155143:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.155162:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000011451c31/0xd5401cbabc020fbc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x813:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22d0 expref: 244 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.155178:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032eb800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+244:2389:x1870153002465408:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.155188:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36427 00010000:00010000:1.0:1783520936.155194:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000011451c31/0xd5401cbabc020fbc lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x813:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22d0 expref: 244 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.155195:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.155206:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943629895850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+244:2388:x1870153002465536:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.155210:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000030e97e7b/0xd5401cbabc020fdf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x814:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22de expref: 244 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.155223:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.155233:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.155247:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000031bd9081/0xd5401cbabc02102c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x815:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22ec expref: 244 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.155252:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002465664 00000100:00100000:2.0:1783520936.155257:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002465664 00010000:00010000:1.0:1783520936.155263:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000011451c31/0xd5401cbabc020fbc lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x813:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22d0 expref: 244 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.155271:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000031bd9081/0xd5401cbabc02102c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x815:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22ec expref: 245 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.155279:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009713add6/0xd5401cbabc022adc lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x816:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31cc expref: 245 pid: 20140 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.155291:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x813:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.155309:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000030e97e7b/0xd5401cbabc020fdf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x814:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22de expref: 244 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.155310:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x813:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.155325:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000031bd9081/0xd5401cbabc02102c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x815:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22ec expref: 244 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.155329:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026694 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x813:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.155331:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36428 00000100:00100000:2.0:1783520936.155377:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513bc0700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+244:2389:x1870153002465664:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.155388:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.155393:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009713add6/0xd5401cbabc022adc lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x816:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31cc expref: 244 pid: 20140 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.155397:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000030e97e7b/0xd5401cbabc020fdf lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x814:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f22de expref: 244 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.155418:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x815:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.155428:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x815:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.155430:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026694 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x813:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.155432:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009713add6/0xd5401cbabc022adc lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x816:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31cc expref: 243 pid: 20140 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.155437:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0266a2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x815:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.155503:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0266a2 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x815:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.155512:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009713add6/0xd5401cbabc022adc lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x816:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31cc expref: 243 pid: 20140 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.155521:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.155535:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x816:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.155538:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x816:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.155542:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0266a9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x816:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.155549:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0266a9 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x816:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.155552:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.155568:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.155574:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032e8700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+242:2388:x1870153002465280:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 452us (616us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.155585:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943629895850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+242:2388:x1870153002465536:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 385us (553us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.155593:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36426 00000100:00100000:2.0:1783520936.155602:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513bc0700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+242:2389:x1870153002465664:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 247us (385us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.155609:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36427 00000100:00100000:2.0:1783520936.155623:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36428 00010000:00010000:3.0:1783520936.155626:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x814:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.155632:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x814:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.155638:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc02669b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x814:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.155643:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0266a2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x815:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.155653:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc02669b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x814:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.155653:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0266a9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x816:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.155673:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993246976:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.155701:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0266b0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x818:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.155713:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0266a2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x815:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.155736:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0266a9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x816:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.155769:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0266a2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x815:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.155772:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0266a9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x816:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.155773:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.155804:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0266a2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x815:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.155805:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032eb800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+238:2389:x1870153002465408:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 774us (953us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.155809:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0266a9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x816:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.155844:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36425 00010000:00010000:1.0:1783520936.155864:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520936.155875:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520936.155887:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520936.155889:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.155894:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0266a2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x815:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.155895:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0266a9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x816:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.155918:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.155921:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.155924:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0266a9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x816:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.155926:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0266a2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x815:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.155946:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0266a9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x816:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.155948:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0266a2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x815:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.155978:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0266a2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x815:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.155982:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0266a9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x816:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.155993:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0266a2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x815:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.155998:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0266a9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x816:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.156018:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.156026:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.156032:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.156037:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507939880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+769:3628:x1870152993246208:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3018us (3191us total) trans 4294976337 rc 0/0 00010000:00010000:0.0:1783520936.156041:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.156046:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.156048:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507939c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+769:3628:x1870152993246080:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3533us (3751us total) trans 4294976336 rc 0/0 00000100:00100000:2.0:1783520936.156050:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44837 00010000:00010000:0.0:1783520936.156052:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.156067:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44836 00010000:00010000:2.0:1783520936.156096:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026694 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x813:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.156108:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026694 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x813:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.156118:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026694 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x813:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.156127:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026694 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x813:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.156128:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513bc3100/2, rc 0, transno 4294976337 00000100:00100000:0.0:1783520936.156135:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993247232:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.156153:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.156163:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993246208:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.156163:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0266b7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x817:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.156198:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350793b100/2, rc 0, transno 4294976336 00000100:00100000:1.0:1783520936.156207:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350793b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993246080:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.156218:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.156242:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.156244:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993247616:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.156246:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002465792 00010000:00010000:3.0:1783520936.156248:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.156258:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026694 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x813:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.156258:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002465792 00010000:00010000:3.0:1783520936.156268:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.156273:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026694 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x813:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.156285:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000094b21e9e/0xd5401cbabc0210f7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x818:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2316 expref: 238 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.156289:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026694 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x813:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.156305:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.156323:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026694 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x813:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.156329:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002465920 00010000:00010000:3.0:1783520936.156336:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026694 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x813:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.156336:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002465920 00000100:00100000:2.0:1783520936.156386:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363fe55c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+768:3625:x1870152993245568:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4714us (4833us total) trans 4294976338 rc 0/0 00010000:00010000:0.0:1783520936.156396:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a5da0c5f/0xd5401cbabc0210aa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x817:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2308 expref: 239 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.156399:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44834 00010000:00010000:3.0:1783520936.156400:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.156406:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520936.156406:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363fe55880/2, rc 0, transno 4294976338 00010000:00010000:3.0:1783520936.156414:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.156440:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363fe55880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993245568:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.156443:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993247744:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.156455:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36429 00000100:00100000:0.0:1783520936.156472:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503370e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+239:2388:x1870153002465792:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.156480:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993247616 00010000:00010000:0.0:1783520936.156483:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.156486:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993247616 00000100:00100000:1.0:1783520936.156501:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:0.0:1783520936.156503:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000094b21e9e/0xd5401cbabc0210f7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x818:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2316 expref: 239 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.156513:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.156518:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993247872:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.156528:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000094b21e9e/0xd5401cbabc0210f7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x818:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2316 expref: 239 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.156532:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44842 00000100:00100000:2.0:1783520936.156548:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507938a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+768:3625:x1870152993247616:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.156555:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000094b21e9e/0xd5401cbabc0210f7 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x818:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2316 expref: 239 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.156562:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2075 count 1 00010000:00010000:0.0:1783520936.156564:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x818:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.156568:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x818:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.156572:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0266b0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x818:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.156577:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0266cc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x81b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.156579:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0266b0 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x818:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.156600:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009fca59cb/0xd5401cbabc02119f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x81b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2340 expref: 238 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.156615:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x81b:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.156615:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.156617:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.156624:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x81b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.156629:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0266cc lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x81b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.156651:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009fca59cb/0xd5401cbabc02119f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x81b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2340 expref: 238 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.156655:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503370e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+238:2388:x1870153002465792:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 185us (505us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.156662:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009fca59cb/0xd5401cbabc02119f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x81b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2340 expref: 238 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.156669:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36429 00000100:00100000:1.0:1783520936.156673:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993247744 00000100:00100000:1.0:1783520936.156685:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993247744 00000100:00100000:2.0:1783520936.156691:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511adaa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993248000:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.156697:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36430 00000100:00100000:0.0:1783520936.156708:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+237:2388:x1870153002465920:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.156719:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.156725:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993247872 00010000:00010000:0.0:1783520936.156727:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a5da0c5f/0xd5401cbabc0210aa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x817:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2308 expref: 237 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.156731:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993247872 00000100:00100000:2.0:1783520936.156734:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607eb8300 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.156748:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a5da0c5f/0xd5401cbabc0210aa lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x817:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2308 expref: 237 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.156760:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44843 00000100:00100000:1.0:1783520936.156779:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44844 00000100:00100000:2.0:1783520936.156780:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+770:3626:x1870152993247744:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.156780:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a5da0c5f/0xd5401cbabc0210aa lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x817:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2308 expref: 237 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.156807:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350793aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+770:3628:x1870152993247872:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.156807:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x817:0x0].0x0 since=0s v=(0 0) 00002000:00080000:2.0:1783520936.156808:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2076 count 1 00000100:00100000:3.0:1783520936.156826:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.156828:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x817:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.156836:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0266d3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x81c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.156837:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0266b7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x817:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.156841:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2077 count 1 00010000:00010000:0.0:1783520936.156864:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0266b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x817:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.156867:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000087b6cc86/0xd5401cbabc0211ec lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x81c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f234e expref: 236 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.156869:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0266da lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x81d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.156915:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.156918:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x81c:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.156923:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.156924:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008b6b9930/0xd5401cbabc02126a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x81d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f235c expref: 236 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.156926:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x81c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.156940:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0266d3 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x81c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.156948:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x81d:0x0].0x0 since=38s v=(0 0) 00000100:00100000:0.0:1783520936.156965:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+236:2388:x1870153002465920:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 253us (743us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.156966:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x81d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.156980:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0266da lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x81d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.156981:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000087b6cc86/0xd5401cbabc0211ec lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x81c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f234e expref: 236 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.156990:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36430 00000100:00100000:0.0:1783520936.157017:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002466048 00000100:00100000:0.0:1783520936.157026:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002466048 00000100:00100000:3.0:1783520936.157028:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503371f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993247488:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.157036:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000087b6cc86/0xd5401cbabc0211ec lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x81c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f234e expref: 236 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.157047:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ad8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993248128:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.157051:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c9f0710e/0xd5401cbabc021167 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x81a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2332 expref: 236 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.157053:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0266be lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x819:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.157064:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008b6b9930/0xd5401cbabc02126a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x81d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f235c expref: 236 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.157077:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514825600 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.157104:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008b6b9930/0xd5401cbabc02126a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x81d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f235c expref: 236 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.157107:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc02669b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x814:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.157112:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002466176 00000100:00100000:0.0:1783520936.157121:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002466176 00010000:00010000:0.0:1783520936.157145:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002bbdf4b5/0xd5401cbabc02112f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x819:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2324 expref: 237 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.157147:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc02669b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x814:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.157152:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507938700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993248256:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.157165:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36431 00010000:00010000:2.0:1783520936.157185:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc02669b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x814:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.157191:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+237:2390:x1870153002466048:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.157196:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc02669b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x814:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.157202:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36432 00010000:00010000:3.0:1783520936.157204:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.157215:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c9f0710e/0xd5401cbabc021167 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x81a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2332 expref: 237 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.157216:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.157216:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503372300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+237:2390:x1870153002466176:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.157218:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.157237:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.157246:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c9f0710e/0xd5401cbabc021167 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x81a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2332 expref: 237 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.157246:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002bbdf4b5/0xd5401cbabc02112f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x819:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2324 expref: 237 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.157273:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002bbdf4b5/0xd5401cbabc02112f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x819:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2324 expref: 237 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.157279:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507938380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993247360:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.157282:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c9f0710e/0xd5401cbabc021167 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x81a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2332 expref: 237 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.157312:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350793a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+771:3626:x1870152993245696:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5343us (5541us total) trans 4294976339 rc 0/0 00010000:00010000:1.0:1783520936.157313:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0266c5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x81a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.157315:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x81a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.157318:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002bbdf4b5/0xd5401cbabc02112f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x819:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2324 expref: 236 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.157336:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x81a:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.157336:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44835 00010000:00010000:0.0:1783520936.157409:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x819:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.157412:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x819:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.157416:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0266be lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x819:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.157423:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0266be lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x819:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.157441:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.157451:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0266c5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.157463:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503372300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+235:2390:x1870153002466176:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 250us (562us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.157464:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0266c5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x81a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.157477:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36432 00010000:00010000:3.0:1783520936.157503:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.157509:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc02669b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x814:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.157519:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.157525:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc02669b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x814:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.157555:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0266b7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x817:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.157570:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0266b7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x817:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.157579:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0266b7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x817:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.157586:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0266b7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x817:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.157595:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.157672:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635739c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+771:3625:x1870152993246592:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3406us (3769us total) trans 4294976340 rc 0/0 00000100:00100000:0.0:1783520936.157677:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44839 00010000:00010000:3.0:1783520936.157701:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc02669b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x814:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.157722:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc02669b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x814:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.157723:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350d165880/2, rc 0, transno 4294976340 00010000:00010000:3.0:1783520936.157741:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc02669b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x814:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.157752:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350d165880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993246592:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.157772:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511adaa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993248000:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.157794:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0266cc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x81b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.157815:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.157819:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0266b7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x817:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.157821:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0266b0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x818:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.157832:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.157842:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0266b7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x817:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.157842:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0266b0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x818:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.157866:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511adaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993248384:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.157867:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0266b0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x818:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.157882:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0266b0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x818:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.157898:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.157908:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520936.157909:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.157914:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0266b0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x818:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.157924:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.157929:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0266b0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x818:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.157934:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0266b7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x817:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.157941:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0266b0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x818:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.157959:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0266b7 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x817:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.157964:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0266b0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x818:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.157969:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc0266b7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x817:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.157980:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.157983:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0266b0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x818:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.157985:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993248384 00000100:00100000:1.0:1783520936.157991:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993248384 00010000:00010000:3.0:1783520936.157992:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.157998:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.158003:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.158016:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.158016:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.158021:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.158029:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.158041:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44845 00000100:00100000:0.0:1783520936.158046:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002466304 00000100:00100000:0.0:1783520936.158053:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002466304 00000100:00100000:1.0:1783520936.158056:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507938380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+771:3628:x1870152993248384:12345-0@lo:6:osp-syn-1-0.0 00000004:00080000:3.0:1783520936.158058:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350544d500/2, rc 0, transno 4294976339 00010000:00010000:0.0:1783520936.158078:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009fca59cb/0xd5401cbabc02119f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x81b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2340 expref: 235 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.158082:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350544d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993245696:0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.158085:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2078 count 1 00010000:00010000:1.0:1783520936.158111:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0266e1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x81e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.158130:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000fcfe98b2/0xd5401cbabc0229cb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x81e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f30bb expref: 235 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.158131:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36433 00010000:00010000:1.0:1783520936.158147:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x81e:0x0].0x0 since=37s v=(0 0) 00000100:00100000:0.0:1783520936.158154:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943511ad8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+235:2387:x1870153002466304:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.158157:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x81e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.158166:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0266e1 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x81e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.158168:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.158180:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.158182:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009fca59cb/0xd5401cbabc02119f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x81b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2340 expref: 235 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.158198:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ad9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993248512:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.158200:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009fca59cb/0xd5401cbabc02119f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x81b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2340 expref: 235 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.158217:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+235:2390:x1870153002466048:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 1031us (1401us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.158230:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36431 00010000:00010000:0.0:1783520936.158230:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009fca59cb/0xd5401cbabc02119f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x81b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2340 expref: 235 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.158248:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x81b:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.158258:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.158259:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x81b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.158268:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0266cc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.158278:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0266cc lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x81b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.158305:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.158316:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993248512 00000100:00100000:1.0:1783520936.158323:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993248512 00000100:00100000:2.0:1783520936.158325:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ad8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993248128:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.158340:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943511ad8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+233:2387:x1870153002466304:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 192us (363us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.158346:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0266d3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x81c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.158351:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36433 00000100:00100000:1.0:1783520936.158410:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44846 00000100:00100000:1.0:1783520936.158428:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350793ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+772:3625:x1870152993248512:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.158438:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0266cc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.158444:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2079 count 1 00010000:00010000:2.0:1783520936.158457:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0266cc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.158466:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc0266e8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x81f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.158469:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0266cc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.158517:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032e9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+773:3626:x1870152993246720:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4448us (4588us total) trans 4294976341 rc 0/0 00010000:00010000:2.0:1783520936.158519:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0266cc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x81b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.158524:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a2259098/0xd5401cbabc021358 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x81f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2378 expref: 232 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.158542:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44838 00010000:00010000:1.0:1783520936.158568:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x81f:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.158573:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x81f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.158578:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc0266e8 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x81f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.158579:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.158582:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.158594:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0266cc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x81b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.158606:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.158610:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0266cc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x81b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.158619:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0266cc lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x81b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.158625:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a2259098/0xd5401cbabc021358 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x81f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2378 expref: 232 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.158639:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0266cc lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x81b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.158645:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a2259098/0xd5401cbabc021358 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x81f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2378 expref: 232 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.158667:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0266cc lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x81b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.158681:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507938a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+773:3625:x1870152993247616:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2134us (2386us total) trans 4294976342 rc 0/0 00000100:00100000:1.0:1783520936.158685:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507939880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993248640:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.158690:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.158695:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.158698:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44842 00010000:00010000:3.0:1783520936.158699:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520936.158729:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350d164000/2, rc 0, transno 4294976341 00000004:00080000:1.0:1783520936.158734:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513bc0380/2, rc 0, transno 4294976342 00000100:00100000:1.0:1783520936.158751:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993247616:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.158752:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350d164000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993246720:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.158757:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.158811:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002466432 00010000:00010000:1.0:1783520936.158813:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0266c5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.158817:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002466432 00000100:00100000:3.0:1783520936.158832:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ad8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993248768:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.158833:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0266c5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.158844:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000087b6cc86/0xd5401cbabc0211ec lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x81c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f234e expref: 233 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.158879:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.158880:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0266c5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.158895:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0266c5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x81a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.158907:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36434 00000100:00100000:0.0:1783520936.158925:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943618f35c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+233:2387:x1870153002466432:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.158935:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.158944:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:0.0:1783520936.158945:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000087b6cc86/0xd5401cbabc0211ec lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x81c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f234e expref: 233 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.158946:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.158959:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ad9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993248896:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.158968:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000087b6cc86/0xd5401cbabc0211ec lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x81c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f234e expref: 233 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.158974:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0266be lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x819:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.158974:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002466560 00000100:00100000:2.0:1783520936.158981:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002466560 00010000:00010000:3.0:1783520936.158985:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0266be lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x819:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.158992:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000087b6cc86/0xd5401cbabc0211ec lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x81c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f234e expref: 234 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.158996:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0266be lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x819:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.159001:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008b6b9930/0xd5401cbabc02126a lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x81d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f235c expref: 234 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.159013:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x81c:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.159020:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.159032:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x81c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.159033:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0266be lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x819:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.159041:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0266d3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.159052:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0266d3 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x81c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.159053:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.159056:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36435 00010000:00010000:3.0:1783520936.159065:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:2.0:1783520936.159067:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+233:2387:x1870153002466560:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.159072:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0266be lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x819:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.159081:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.159081:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.159087:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.159089:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0266be lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x819:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.159091:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008b6b9930/0xd5401cbabc02126a lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x81d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f235c expref: 233 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.159105:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0266be lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x819:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.159105:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008b6b9930/0xd5401cbabc02126a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x81d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f235c expref: 233 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.159121:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x81d:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.159122:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943618f35c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+233:2387:x1870153002466432:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 199us (367us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.159125:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0266be lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x819:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.159125:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000fcfe98b2/0xd5401cbabc0229cb lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x81e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30bb expref: 233 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.159129:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x81d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.159146:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0266da lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.159153:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36434 00010000:00010000:3.0:1783520936.159175:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc0266be lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x819:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.159176:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0266da lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x81d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.159183:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000fcfe98b2/0xd5401cbabc0229cb lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x81e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30bb expref: 232 pid: 9151 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.159205:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.159206:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993248768 00010000:00010000:3.0:1783520936.159212:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.159212:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.159213:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993248768 00010000:00010000:3.0:1783520936.159216:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.159219:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350793a680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993249024:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.159240:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993248896 00000100:00100000:2.0:1783520936.159243:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+232:2387:x1870153002466560:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 174us (298us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.159245:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993248896 00000100:00100000:2.0:1783520936.159256:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36435 00000100:00100000:0.0:1783520936.159280:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44847 00000100:00100000:0.0:1783520936.159298:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+775:3626:x1870152993248768:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.159302:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0266d3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.159304:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470900 going to sleep for 11 seconds 00002000:00080000:0.0:1783520936.159314:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2080 count 1 00010000:00010000:2.0:1783520936.159321:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0266d3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.159332:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0266d3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.159335:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0266ef lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x820:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.159337:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350793bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+775:3627:x1870152993246848:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4879us (5160us total) trans 4294976343 rc 0/0 00010000:00010000:2.0:1783520936.159346:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0266d3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x81c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.159352:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44840 00000004:00080000:1.0:1783520936.159355:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435032eb480/2, rc 0, transno 4294976343 00010000:00010000:0.0:1783520936.159435:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001c8c729e/0xd5401cbabc0213ba lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x820:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2386 expref: 231 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.159444:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032eb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993246848:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.159447:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.159447:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x820:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.159461:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x820:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.159467:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0266ef lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x820:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.159501:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:0.0:1783520936.159507:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520936.159514:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0266d3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x81c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.159527:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943511ad8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993249152:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.159530:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.159541:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0266d3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x81c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.159553:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0266d3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x81c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.159568:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0266d3 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x81c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.159579:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0266d3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x81c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.159581:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.159583:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.159617:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.159621:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.159624:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.159635:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+775:3626:x1870152993247744:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2864us (3126us total) trans 4294976344 rc 0/0 00000100:00100000:0.0:1783520936.159644:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993249152 00000100:00100000:2.0:1783520936.159649:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44843 00000100:00100000:0.0:1783520936.159651:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993249152 00000100:00100000:2.0:1783520936.159683:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44848 00010000:00010000:3.0:1783520936.159689:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0266c5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x81a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.159690:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44849 00000100:00100000:2.0:1783520936.159694:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507939c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+775:3628:x1870152993248896:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.159701:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb800 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520936.159706:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363fe54a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+775:3628:x1870152993249152:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.159707:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0266c5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x81a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.159708:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2081 count 1 00010000:00010000:3.0:1783520936.159720:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0266c5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x81a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.159723:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0266f6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x821:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.159725:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2082 count 1 00010000:00010000:2.0:1783520936.159739:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005d191631/0xd5401cbabc021400 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x821:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2394 expref: 231 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.159741:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0266fd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x822:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.159743:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0266c5 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x81a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.159752:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x821:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.159759:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x821:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.159763:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0266c5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x81a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.159767:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0266f6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x821:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.159773:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000034b1c965/0xd5401cbabc02142a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x822:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f23a2 expref: 231 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.159784:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.159789:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.159790:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x822:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.159794:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.159795:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x822:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.159798:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005d191631/0xd5401cbabc021400 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x821:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2394 expref: 231 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.159801:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0266fd lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x822:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.159822:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005d191631/0xd5401cbabc021400 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x821:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2394 expref: 231 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.159825:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507939180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+776:3627:x1870152993247104:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5337us (5521us total) trans 4294976345 rc 0/0 00000004:00080000:3.0:1783520936.159838:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513bc3480/2, rc 0, transno 4294976344 00010000:00010000:0.0:1783520936.159838:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000034b1c965/0xd5401cbabc02142a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x822:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23a2 expref: 231 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.159859:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44841 00000100:00100000:3.0:1783520936.159864:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993247744:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.159873:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993249280:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.159875:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000034b1c965/0xd5401cbabc02142a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x822:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23a2 expref: 231 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.159892:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993249408:192.168.204.33@tcp:104: 00000004:00080000:1.0:1783520936.159925:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435032eaa00/2, rc 0, transno 4294976345 00000100:00100000:2.0:1783520936.159938:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351dc40700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993249536:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.159949:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993247104:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.159950:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514219780 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.159980:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.159981:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001c8c729e/0xd5401cbabc0213ba lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x820:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2386 expref: 231 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.159994:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001c8c729e/0xd5401cbabc0213ba lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x820:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2386 expref: 231 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.160007:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993249536 00000100:00100000:0.0:1783520936.160009:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993249664:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.160014:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993249536 00000100:00100000:2.0:1783520936.160014:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1783520936.160024:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363fe55180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993249792:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.160042:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350b83ef00 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.160063:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.160070:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993249792 00000100:00100000:3.0:1783520936.160071:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44850 00010000:00010000:1.0:1783520936.160072:0:20415:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008b6b9930/0xd5401cbabc02126a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x81d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f235c expref: 231 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.160076:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993249792 00000100:00100000:3.0:1783520936.160087:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e472300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+777:3625:x1870152993249536:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.160107:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44851 00002000:00080000:3.0:1783520936.160116:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2083 count 1 00000100:00100000:1.0:1783520936.160120:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507938700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993248256:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.160121:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e471f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+777:3625:x1870152993249792:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.160131:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026704 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x823:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.160141:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2084 count 1 00010000:00010000:3.0:1783520936.160144:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002844f84e/0xd5401cbabc021485 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x823:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f23b0 expref: 230 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.160151:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x823:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.160152:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02670b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x824:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.160154:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x823:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.160155:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0266da lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.160164:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026704 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x823:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.160168:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000079379aa7/0xd5401cbabc021503 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x824:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f23be expref: 230 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.160187:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0266da lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.160188:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x824:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.160199:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x824:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.160202:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002844f84e/0xd5401cbabc021485 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x823:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23b0 expref: 230 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.160213:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02670b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x824:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.160231:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0266da lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.160238:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002844f84e/0xd5401cbabc021485 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x823:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23b0 expref: 230 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.160242:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000079379aa7/0xd5401cbabc021503 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x824:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23be expref: 230 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.160255:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0266da lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.160262:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503372a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993249920:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.160262:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000079379aa7/0xd5401cbabc021503 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x824:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23be expref: 230 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.160288:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0266da lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x81d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.160291:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6600 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.160293:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a5180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993250048:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.160324:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514825600 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.160330:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.160339:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.160345:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0266da lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x81d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.160351:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a87a80 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.160353:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.160379:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0266da lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x81d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.160390:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0266da lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x81d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.160403:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0266da lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x81d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.160411:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0266da lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x81d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.160434:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.160437:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.160439:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.160487:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470e40 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.160507:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350793aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+778:3628:x1870152993247872:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3693us (3887us total) trans 4294976346 rc 0/0 00000100:00100000:1.0:1783520936.160519:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44844 00000004:00080000:1.0:1783520936.160573:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513bc1180/2, rc 0, transno 4294976346 00000100:00100000:1.0:1783520936.160585:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993247872:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.160656:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363fe56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993250176:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.160686:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.160729:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993250176 00000100:00100000:0.0:1783520936.160735:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993250176 00000100:00100000:0.0:1783520936.160765:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44852 00000100:00100000:0.0:1783520936.160772:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363fe54000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+778:3627:x1870152993250176:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.160783:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2085 count 1 00010000:00010000:0.0:1783520936.160794:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026712 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x825:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.160805:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000000e9c9e68/0xd5401cbabc0215b9 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x825:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f23cc expref: 230 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.160813:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x825:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.160816:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x825:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.160820:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026712 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x825:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.160839:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000000e9c9e68/0xd5401cbabc0215b9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x825:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23cc expref: 230 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.160848:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000000e9c9e68/0xd5401cbabc0215b9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x825:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23cc expref: 230 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.160858:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993250304:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.160892:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895840 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.161868:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.161887:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350793a680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993249024:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.161921:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0266e1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x81e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.161931:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993249408:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.161966:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0266fd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x822:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.161974:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002466816 00000100:00100000:3.0:1783520936.161980:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002466816 00000100:00100000:0.0:1783520936.161998:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993249664:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.162011:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0266ef lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x820:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.162044:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000034b1c965/0xd5401cbabc02142a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x822:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23a2 expref: 231 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.162075:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36436 00000100:00100000:3.0:1783520936.162083:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503370380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+231:2388:x1870153002466816:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.162098:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.162103:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000034b1c965/0xd5401cbabc02142a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x822:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23a2 expref: 231 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.162113:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000034b1c965/0xd5401cbabc02142a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x822:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23a2 expref: 231 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.162129:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000034b1c965/0xd5401cbabc02142a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x822:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23a2 expref: 231 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.162139:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x822:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.162143:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x822:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.162148:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0266fd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x822:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.162162:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0266fd lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x822:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.162183:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.162204:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0266fd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x822:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.162215:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0266fd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x822:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.162226:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0266fd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x822:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.162234:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0266fd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x822:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.162255:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520936.162289:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503370380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+230:2388:x1870153002466816:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 205us (424us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.162297:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36436 00010000:00010000:3.0:1783520936.162319:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.162326:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0266fd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x822:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.162334:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.162339:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0266fd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x822:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.162348:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0266fd lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x822:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.162423:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363fe54a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+779:3628:x1870152993249152:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2717us (2840us total) trans 4294976347 rc 0/0 00000100:00100000:0.0:1783520936.162434:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44849 00010000:00010000:3.0:1783520936.162442:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0266fd lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x822:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.162453:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943511ad8000/2, rc 0, transno 4294976347 00010000:00010000:3.0:1783520936.162454:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0266fd lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x822:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.162477:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ad8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993249152:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.162481:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.162487:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a5180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993250048:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.162503:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.162511:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.162526:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02670b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x824:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.162546:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503372a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993249920:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.162567:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:3.0:1783520936.162573:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026704 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x823:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.162584:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363fe57800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993250432:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.162595:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993249280:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.162616:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0266f6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x821:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.162632:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.162652:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.162697:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993250432 00000100:00100000:2.0:1783520936.162699:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.162706:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993250432 00000100:00100000:0.0:1783520936.162727:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002466688 00000100:00100000:0.0:1783520936.162736:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002466688 00000100:00100000:2.0:1783520936.162742:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002466944 00000100:00100000:2.0:1783520936.162748:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002466944 00000100:00100000:1.0:1783520936.162760:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44853 00010000:00010000:0.0:1783520936.162768:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000fcfe98b2/0xd5401cbabc0229cb lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x81e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30bb expref: 231 pid: 9151 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.162772:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001c8c729e/0xd5401cbabc0213ba lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x820:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2386 expref: 231 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.162779:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363fe55c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+779:3627:x1870152993250432:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.162823:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2086 count 1 00000100:00100000:0.0:1783520936.162834:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36437 00000100:00100000:2.0:1783520936.162844:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36438 00000100:00100000:0.0:1783520936.162850:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a5180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+231:2389:x1870153002466688:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.162856:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026719 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x826:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.162858:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+231:2388:x1870153002466944:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.162865:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.162881:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000fcfe98b2/0xd5401cbabc0229cb lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x81e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30bb expref: 231 pid: 9151 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.162882:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.162894:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001c8c729e/0xd5401cbabc0213ba lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x820:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2386 expref: 231 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.162896:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f5600e2d/0xd5401cbabc0228ba lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x826:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2faa expref: 231 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.162905:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000fcfe98b2/0xd5401cbabc0229cb lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x81e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30bb expref: 231 pid: 9151 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.162939:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x826:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.162957:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x826:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.162968:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026719 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x826:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.162971:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000fcfe98b2/0xd5401cbabc0229cb lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x81e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30bb expref: 231 pid: 9151 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.162998:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001c8c729e/0xd5401cbabc0213ba lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x820:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2386 expref: 231 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.163025:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x81e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.163037:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f5600e2d/0xd5401cbabc0228ba lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x826:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2faa expref: 230 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.163038:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001c8c729e/0xd5401cbabc0213ba lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x820:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2386 expref: 230 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.163038:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x81e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.163060:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0266e1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.163087:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f5600e2d/0xd5401cbabc0228ba lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x826:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2faa expref: 230 pid: 178496 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.163092:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x820:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.163092:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0266e1 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x81e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.163112:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x820:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.163125:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0266ef lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x820:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.163134:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.163135:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363fe56d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993250560:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.163139:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0266ef lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x820:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.163181:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.163181:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a5180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+229:2389:x1870153002466688:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 334us (527us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.163192:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36437 00000100:00100000:2.0:1783520936.163221:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+228:2388:x1870153002466944:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 362us (517us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.163233:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36438 00000100:00100000:1.0:1783520936.163240:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470900 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.163242:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0266ef lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x820:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.163257:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0266ef lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x820:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.163267:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0266ef lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x820:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.163276:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0266ef lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x820:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.163285:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0266e1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.163292:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.163309:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.163311:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.163326:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0266e1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.163327:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0266ef lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x820:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.163343:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002467200 00000100:00100000:2.0:1783520936.163353:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002467200 00010000:00010000:0.0:1783520936.163403:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.163415:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0266ef lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x820:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.163419:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0266e1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.163422:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000079379aa7/0xd5401cbabc021503 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x824:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23be expref: 228 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.163468:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0266ef lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x820:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.163512:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0266e1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x81e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.163514:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36439 00010000:00010000:0.0:1783520936.163520:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0266ef lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x820:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.163541:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435161c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+228:2388:x1870153002467200:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.163568:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0266ef lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x820:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.163569:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.163577:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.163579:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000079379aa7/0xd5401cbabc021503 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x824:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23be expref: 228 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.163586:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.163592:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0266e1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x81e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.163600:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.163601:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000079379aa7/0xd5401cbabc021503 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x824:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23be expref: 228 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.163604:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0266e1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x81e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.163623:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0266e1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x81e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.163628:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000079379aa7/0xd5401cbabc021503 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x824:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23be expref: 228 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.163645:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0266e1 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x81e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.163646:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x824:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.163654:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x824:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.163664:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02670b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x824:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.163665:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc0266e1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x81e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.163687:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02670b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x824:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.163699:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.163703:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.163708:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.163711:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.163730:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507938380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+780:3628:x1870152993248384:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5677us (5819us total) trans 4294976348 rc 0/0 00000100:00100000:1.0:1783520936.163742:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44845 00010000:00010000:3.0:1783520936.163750:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.163758:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435161c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+227:2388:x1870153002467200:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 222us (460us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.163770:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36439 00000004:00080000:1.0:1783520936.163812:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943511adaa00/2, rc 0, transno 4294976348 00010000:00010000:2.0:1783520936.163819:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02670b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x824:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.163828:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511adaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993248384:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.163849:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02670b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x824:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.163855:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+780:3626:x1870152993248768:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4562us (4980us total) trans 4294976349 rc 0/0 00010000:00010000:2.0:1783520936.163863:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02670b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x824:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.163876:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44847 00010000:00010000:2.0:1783520936.163879:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02670b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x824:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.163907:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350793a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993250688:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.163908:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.163961:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520936.163961:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.163974:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02670b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x824:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.163986:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.163993:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02670b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x824:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.164003:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02670b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x824:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.164016:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02670b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x824:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.164023:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc02670b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x824:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.164035:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993250688 00010000:00010000:3.0:1783520936.164041:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.164043:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.164045:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993250688 00010000:00010000:3.0:1783520936.164046:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.164058:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e471f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+781:3625:x1870152993249792:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3939us (3992us total) trans 4294976350 rc 0/0 00000100:00100000:2.0:1783520936.164072:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44851 00000004:00080000:3.0:1783520936.164087:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943511ad8700/2, rc 0, transno 4294976349 00000004:00080000:1.0:1783520936.164099:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363fe55180/2, rc 0, transno 4294976350 00000100:00100000:1.0:1783520936.164114:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363fe55180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993249792:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.164148:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1783520936.164160:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514fdf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993250816:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.164197:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.164209:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.164269:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.164289:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993250816 00000100:00100000:1.0:1783520936.164297:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993250816 00000100:00100000:3.0:1783520936.164302:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ad8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993248768:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.164308:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002467072 00000100:00100000:0.0:1783520936.164313:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002467072 00000100:00100000:3.0:1783520936.164331:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002467328 00010000:00010000:0.0:1783520936.164338:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002844f84e/0xd5401cbabc021485 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x823:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23b0 expref: 227 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.164339:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002467328 00000100:00100000:2.0:1783520936.164383:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.164418:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514fded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993250944:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.164428:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.164440:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002467456 00000100:00100000:0.0:1783520936.164445:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002467456 00010000:00010000:3.0:1783520936.164453:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005d191631/0xd5401cbabc021400 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x821:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2394 expref: 229 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.164465:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a2259098/0xd5401cbabc021358 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x81f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2378 expref: 229 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.164479:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002467584 00000100:00100000:1.0:1783520936.164484:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.164499:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002467584 00010000:00010000:3.0:1783520936.164526:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000000e9c9e68/0xd5401cbabc0215b9 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x825:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23cc expref: 230 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.164531:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36440 00000100:00100000:0.0:1783520936.164545:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b4000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+230:2388:x1870153002467072:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.164558:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.164566:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36441 00010000:00010000:0.0:1783520936.164568:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002844f84e/0xd5401cbabc021485 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x823:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23b0 expref: 230 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.164581:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.164582:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943618f37850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+230:2388:x1870153002467328:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.164592:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.164598:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002844f84e/0xd5401cbabc021485 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x823:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23b0 expref: 230 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.164599:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005d191631/0xd5401cbabc021400 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x821:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2394 expref: 230 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.164606:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363fe56d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993250560:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.164613:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002467712 00000100:00100000:2.0:1783520936.164625:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002467712 00010000:00010000:0.0:1783520936.164640:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002844f84e/0xd5401cbabc021485 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x823:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23b0 expref: 231 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.164651:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026719 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x826:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.164652:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005d191631/0xd5401cbabc021400 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x821:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2394 expref: 231 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.164654:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f5600e2d/0xd5401cbabc0228ba lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x826:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2faa expref: 231 pid: 178496 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.164666:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x823:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.164687:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x823:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.164714:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026704 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x823:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.164725:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026704 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x823:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.164727:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005d191631/0xd5401cbabc021400 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x821:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2394 expref: 230 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.164739:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993250944 00000100:00100000:2.0:1783520936.164741:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36442 00000100:00100000:1.0:1783520936.164748:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993250944 00010000:00010000:3.0:1783520936.164754:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x821:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.164760:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x821:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.164760:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b4a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+229:2388:x1870153002467456:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.164769:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0266f6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x821:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.164777:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0266f6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x821:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.164778:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.164786:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.164789:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a2259098/0xd5401cbabc021358 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x81f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2378 expref: 229 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.164798:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.164803:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44854 00010000:00010000:2.0:1783520936.164806:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a2259098/0xd5401cbabc021358 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x81f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2378 expref: 229 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.164819:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507938700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+782:3625:x1870152993250688:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.164826:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x81f:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.164835:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943618f37850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+229:2388:x1870153002467328:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 253us (560us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.164835:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b4000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+229:2388:x1870153002467072:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 285us (623us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.164836:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x81f:0x0].0x0 since=0s v=(0 0) 00002000:00080000:1.0:1783520936.164841:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2087 count 1 00010000:00010000:2.0:1783520936.164847:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc0266e8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.164861:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36441 00010000:00010000:2.0:1783520936.164868:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc0266e8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x81f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.164870:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026720 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x827:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.164879:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36440 00000100:00100000:3.0:1783520936.164900:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36443 00010000:00010000:2.0:1783520936.164909:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.164922:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000efe92229/0xd5401cbabc02179c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x827:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f23e8 expref: 227 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.164923:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b6680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+227:2390:x1870153002467584:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.164942:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b4a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+227:2388:x1870153002467456:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 180us (574us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.164943:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36444 00010000:00010000:3.0:1783520936.164945:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.164956:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000000e9c9e68/0xd5401cbabc0215b9 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x825:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23cc expref: 227 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.164974:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x827:0x0].0x0 since=38s v=(0 0) 00000100:00100000:0.0:1783520936.164976:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b4700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+227:2387:x1870153002467712:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.164981:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36442 00010000:00010000:1.0:1783520936.164991:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x827:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.164999:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.165000:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026720 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x827:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.165006:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000000e9c9e68/0xd5401cbabc0215b9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x825:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23cc expref: 226 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.165016:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f5600e2d/0xd5401cbabc0228ba lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x826:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2faa expref: 226 pid: 178496 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.165018:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0266f6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x821:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.165060:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0266f6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x821:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.165065:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x825:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.165071:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000efe92229/0xd5401cbabc02179c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x827:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23e8 expref: 226 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.165072:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f5600e2d/0xd5401cbabc0228ba lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x826:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2faa expref: 226 pid: 178496 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.165074:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x825:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.165077:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0266f6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x821:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.165081:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026712 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x825:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.165134:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026712 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x825:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.165137:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0266f6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x821:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.165141:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000efe92229/0xd5401cbabc02179c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x827:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23e8 expref: 226 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.165152:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f5600e2d/0xd5401cbabc0228ba lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x826:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2faa expref: 226 pid: 178496 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.165212:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514fdf480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993251072:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.165219:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x826:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.165221:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.165226:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.165227:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x826:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.165233:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0266f6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x821:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.165235:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026719 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x826:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.165255:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.165262:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0266f6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x821:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.165265:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026719 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x826:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.165270:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470900 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.165289:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0266f6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x821:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.165304:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0266f6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x821:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.165311:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0266f6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x821:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.165312:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.165338:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44855 00010000:00010000:3.0:1783520936.165340:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.165348:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.165353:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514fdd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+783:3627:x1870152993250944:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.165354:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.165390:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b4700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+225:2387:x1870153002467712:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 383us (787us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.165397:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507939c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+783:3628:x1870152993248896:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5705us (6387us total) trans 4294976351 rc 0/0 00010000:00010000:3.0:1783520936.165399:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.165408:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36444 00000100:00100000:2.0:1783520936.165414:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44848 00000004:00080000:1.0:1783520936.165437:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943511ad9c00/2, rc 0, transno 4294976351 00000100:00100000:3.0:1783520936.165440:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b6680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+224:2390:x1870153002467584:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 521us (1012us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.165448:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44856 00000100:00100000:3.0:1783520936.165454:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36443 00000100:00100000:1.0:1783520936.165459:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ad9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993248896:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.165464:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514fdc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+782:3625:x1870152993250816:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.165521:0:9151:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a2259098/0xd5401cbabc021358 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x81f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2378 expref: 223 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.165543:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363fe56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993251200:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.165549:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507939880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993248640:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.165581:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc0266e8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.165593:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc0266e8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.165594:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00080000:2.0:1783520936.165598:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2088 count 1 00010000:00010000:0.0:1783520936.165612:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc0266e8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.165622:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc0266e8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x81f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.165625:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026727 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x828:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.165627:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993251200 00000100:00100000:3.0:1783520936.165633:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993251200 00010000:00010000:0.0:1783520936.165637:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc0266e8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x81f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.165656:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000624178c1/0xd5401cbabc02184b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x828:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f23f6 expref: 222 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.165665:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x828:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.165665:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026719 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x826:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.165665:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.165682:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x828:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.165684:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.165697:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026719 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x826:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.165700:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026727 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x828:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.165700:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc0266e8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x81f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.165714:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44857 00000100:00100000:3.0:1783520936.165736:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+783:3628:x1870152993251200:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.165750:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.165752:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026719 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x826:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.165757:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc0266e8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x81f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.165762:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2090 count 1 00010000:00010000:2.0:1783520936.165778:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000624178c1/0xd5401cbabc02184b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x828:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23f6 expref: 222 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.165793:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc02672e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x82a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.165793:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026719 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x826:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.165805:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc0266e8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x81f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.165826:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000624178c1/0xd5401cbabc02184b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x828:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23f6 expref: 222 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.165863:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ecd8f91e/0xd5401cbabc02198d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x82a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2412 expref: 222 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.165865:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc0266e8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x81f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.165870:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b4000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993251328:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.165873:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.165893:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x82a:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.165893:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc0266e8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x81f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.165899:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x82a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.165907:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc02672e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x82a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.165920:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.165924:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.165925:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026719 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x826:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.165928:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.165933:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.165934:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbbe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.165938:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026719 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x826:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.165947:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026719 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x826:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.165952:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514825600 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.165958:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026719 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x826:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.165967:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026719 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x826:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.165976:0:35275:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000e9c9e68/0xd5401cbabc0215b9 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x825:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23cc expref: 222 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.165986:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.165992:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.165993:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993250304:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.165997:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.166019:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026704 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x823:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.166023:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026712 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x825:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.166029:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026704 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x823:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.166033:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363fe55c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+784:3627:x1870152993250432:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3257us (3397us total) trans 4294976352 rc 0/0 00010000:00010000:0.0:1783520936.166072:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026712 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x825:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.166073:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44853 00010000:00010000:3.0:1783520936.166075:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026704 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x823:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.166091:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026712 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x825:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.166102:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026712 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x825:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.166112:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026712 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x825:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.166125:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026704 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x823:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.166128:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363fe57800/2, rc 0, transno 4294976352 00010000:00010000:0.0:1783520936.166136:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.166143:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363fe57800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993250432:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.166145:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.166150:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026712 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x825:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.166156:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.166159:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026712 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x825:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.166171:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026712 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x825:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.166181:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:3.0:1783520936.166183:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026712 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x825:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.166191:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026712 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x825:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.166194:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993251456:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.166224:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.166226:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.166231:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.166247:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.166257:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520936.166261:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363fe54000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+784:3627:x1870152993250176:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5487us (5571us total) trans 4294976353 rc 0/0 00010000:00010000:3.0:1783520936.166268:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026704 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x823:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.166274:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.166288:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5e00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520936.166288:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44852 00010000:00010000:3.0:1783520936.166293:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026704 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x823:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.166301:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026704 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x823:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.166311:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026704 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x823:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.166317:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026704 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x823:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.166325:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993251456 00000100:00100000:0.0:1783520936.166332:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993251456 00010000:00010000:3.0:1783520936.166384:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.166389:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.166391:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520936.166396:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363fe56a00/2, rc 0, transno 4294976353 00000100:00100000:0.0:1783520936.166408:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44858 00000100:00100000:1.0:1783520936.166415:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363fe56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993250176:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.166424:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+784:3626:x1870152993251456:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.166443:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2091 count 1 00010000:00010000:0.0:1783520936.166457:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026735 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x82b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.166479:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c833071b/0xd5401cbabc02199b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x82b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2420 expref: 221 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.166499:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993251584:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.166504:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x82b:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.166511:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x82b:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.166513:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e472300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+785:3625:x1870152993249536:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6427us (6534us total) trans 4294976354 rc 0/0 00010000:00010000:0.0:1783520936.166518:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026735 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x82b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.166526:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44850 00000100:00100000:1.0:1783520936.166545:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.166556:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c833071b/0xd5401cbabc02199b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x82b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2420 expref: 221 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.166571:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c833071b/0xd5401cbabc02199b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x82b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2420 expref: 221 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.166572:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993251584 00000100:00100000:3.0:1783520936.166578:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993251584 00000100:00100000:0.0:1783520936.166591:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993251712:192.168.204.33@tcp:104: 00000004:00080000:1.0:1783520936.166616:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351dc40700/2, rc 0, transno 4294976354 00000100:00100000:3.0:1783520936.166624:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44859 00000100:00100000:1.0:1783520936.166640:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351dc40700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993249536:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.166641:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+785:3626:x1870152993251584:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.166652:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514219780 going to sleep for 11 seconds 00002000:00080000:3.0:1783520936.166676:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2092 count 1 00010000:00010000:3.0:1783520936.166692:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02673c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x82c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.166707:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009d5b7649/0xd5401cbabc0219a9 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x82c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f242e expref: 221 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.166717:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x82c:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.166720:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x82c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.166726:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02673c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x82c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.166730:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1783520936.166744:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993251840:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.166757:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009d5b7649/0xd5401cbabc0219a9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x82c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f242e expref: 221 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.166787:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350793ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+786:3625:x1870152993248512:12345-0@lo:6:osp-syn-1-0.0 Request processed in 8360us (8533us total) trans 4294976355 rc 0/0 00000100:00100000:1.0:1783520936.166792:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.166806:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44846 00010000:00010000:3.0:1783520936.166822:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009d5b7649/0xd5401cbabc0219a9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x82c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f242e expref: 221 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.166838:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993251840 00000100:00100000:0.0:1783520936.166843:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993251840 00000100:00100000:3.0:1783520936.166844:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503372a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993251968:192.168.204.33@tcp:104: 00000004:00080000:1.0:1783520936.166861:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943511ad9500/2, rc 0, transno 4294976355 00000100:00100000:0.0:1783520936.166875:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44860 00000100:00100000:1.0:1783520936.166878:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943511ad9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993248512:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.166896:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+786:3628:x1870152993251840:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.166897:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6600 going to sleep for 11 seconds 00002000:00080000:0.0:1783520936.166917:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2093 count 1 00010000:00010000:0.0:1783520936.166934:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026743 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x82d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.166947:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009dea7d49/0xd5401cbabc0219b7 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x82d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f243c expref: 221 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.166952:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993252096:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.166969:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x82d:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.166972:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x82d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.166978:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026743 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x82d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.167003:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.167004:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ecd8f91e/0xd5401cbabc02198d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x82a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2412 expref: 221 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.167010:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009dea7d49/0xd5401cbabc0219b7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x82d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f243c expref: 221 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.167026:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009dea7d49/0xd5401cbabc0219b7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x82d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f243c expref: 221 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.167039:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993252352:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.167063:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993252096 00010000:00010000:3.0:1783520936.167071:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ecd8f91e/0xd5401cbabc02198d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x82a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2412 expref: 221 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.167073:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993252096 00000100:00100000:3.0:1783520936.167086:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503371500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993252224:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.167092:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895840 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.167128:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44861 00000100:00100000:3.0:1783520936.167140:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6cc0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.167141:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+787:3628:x1870152993252096:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.167160:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2089 count 1 00002000:00080000:0.0:1783520936.167164:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2094 count 1 00010000:00010000:1.0:1783520936.167182:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02674a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x829:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.167184:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026751 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x82e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.167239:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004f5f5a59/0xd5401cbabc022b14 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x82e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3204 expref: 221 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.167241:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000038ed2a55/0xd5401cbabc021939 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x829:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2404 expref: 221 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.167277:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x82e:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.167279:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x829:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.167292:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x82e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.167293:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x829:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.167316:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02674a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x829:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.167317:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026751 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x82e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.167406:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004f5f5a59/0xd5401cbabc022b14 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x82e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3204 expref: 221 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.167421:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000038ed2a55/0xd5401cbabc021939 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x829:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2404 expref: 221 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.167424:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004f5f5a59/0xd5401cbabc022b14 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x82e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3204 expref: 221 pid: 20415 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.167465:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000038ed2a55/0xd5401cbabc021939 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x829:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2404 expref: 221 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.167469:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993252480:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.167511:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a5f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993252608:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.167537:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff9436298a6480 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.167560:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470840 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.169177:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514fdf480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993251072:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.169206:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026720 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x827:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.169729:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a5f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993252608:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.169762:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02674a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x829:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.169770:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993251712:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.169806:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503372a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993251968:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.169815:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026735 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x82b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.169829:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02673c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x82c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.169856:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993252352:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.169868:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503371500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993252224:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.169878:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026743 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x82d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.169888:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc02672e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x82a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.169898:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b4000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993251328:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.169912:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993252480:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.169917:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026727 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x828:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.169929:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026751 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x82e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.170578:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.170602:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.170618:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.170657:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.170675:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.170691:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.170718:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.170737:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002468480 00000100:00100000:3.0:1783520936.170739:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002468224 00000100:00100000:0.0:1783520936.170744:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002468480 00000100:00100000:3.0:1783520936.170747:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002468224 00000100:00100000:2.0:1783520936.170752:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.170773:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002468736 00000100:00100000:2.0:1783520936.170777:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002468736 00010000:00010000:3.0:1783520936.170782:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c833071b/0xd5401cbabc02199b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x82b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2420 expref: 223 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.170785:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009dea7d49/0xd5401cbabc0219b7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x82d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f243c expref: 224 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.170801:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004f5f5a59/0xd5401cbabc022b14 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x82e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3204 expref: 224 pid: 20415 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.170820:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002467968 00000100:00100000:0.0:1783520936.170823:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002468352 00000100:00100000:3.0:1783520936.170825:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002467968 00000100:00100000:2.0:1783520936.170828:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002467840 00000100:00100000:0.0:1783520936.170829:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002468352 00000100:00100000:2.0:1783520936.170835:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002467840 00010000:00010000:3.0:1783520936.170843:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000038ed2a55/0xd5401cbabc021939 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x829:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2404 expref: 227 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.170851:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009d5b7649/0xd5401cbabc0219a9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x82c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f242e expref: 227 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.170856:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000efe92229/0xd5401cbabc02179c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x827:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23e8 expref: 227 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.170860:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002468096 00000100:00100000:3.0:1783520936.170867:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002468096 00000100:00100000:0.0:1783520936.170881:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002468608 00010000:00010000:3.0:1783520936.170886:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000624178c1/0xd5401cbabc02184b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x828:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23f6 expref: 228 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.170887:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002468608 00000100:00100000:2.0:1783520936.170891:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36445 00010000:00010000:0.0:1783520936.170908:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ecd8f91e/0xd5401cbabc02198d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x82a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2412 expref: 229 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.170909:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517208e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+229:2389:x1870153002468224:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520936.170915:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36446 00010000:00010000:2.0:1783520936.170930:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.170931:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b4000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+229:2389:x1870153002468480:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.170940:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36447 00010000:00010000:2.0:1783520936.170941:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c833071b/0xd5401cbabc02199b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x82b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2420 expref: 229 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.170944:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.170955:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009dea7d49/0xd5401cbabc0219b7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x82d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f243c expref: 229 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.170956:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517209500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+229:2389:x1870153002468736:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.170967:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c833071b/0xd5401cbabc02199b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x82b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2420 expref: 229 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.170970:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.170977:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009dea7d49/0xd5401cbabc0219b7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x82d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f243c expref: 229 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.170979:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004f5f5a59/0xd5401cbabc022b14 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x82e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3204 expref: 229 pid: 20415 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.171006:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004f5f5a59/0xd5401cbabc022b14 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x82e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3204 expref: 229 pid: 20415 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.171026:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009dea7d49/0xd5401cbabc0219b7 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x82d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f243c expref: 229 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.171027:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c833071b/0xd5401cbabc02199b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x82b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2420 expref: 229 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.171027:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004f5f5a59/0xd5401cbabc022b14 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x82e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3204 expref: 229 pid: 20415 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.171060:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x82d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.171064:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x82b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.171064:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x82e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.171067:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x82d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.171075:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x82b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.171076:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x82e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.171079:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026743 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x82d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.171090:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026735 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x82b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.171090:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026751 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x82e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.171093:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026743 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x82d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.171118:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026735 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x82b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.171121:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026751 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x82e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.171134:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.171152:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.171159:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.171182:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b4000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+226:2389:x1870153002468480:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 254us (604us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.171191:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517209500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+226:2389:x1870153002468736:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 239us (568us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.171199:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517208e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+226:2389:x1870153002468224:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 291us (591us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.171204:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36446 00000100:00100000:0.0:1783520936.171214:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36447 00000100:00100000:2.0:1783520936.171223:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36445 00000100:00100000:3.0:1783520936.171232:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36448 00000100:00100000:0.0:1783520936.171241:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36449 00000100:00100000:3.0:1783520936.171246:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94361b4a6450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+223:2387:x1870153002467968:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.171251:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36450 00000100:00100000:0.0:1783520936.171256:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351720b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+223:2387:x1870153002468352:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.171261:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.171267:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517208000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+223:2387:x1870153002467840:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.171267:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.171270:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000038ed2a55/0xd5401cbabc021939 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x829:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2404 expref: 223 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.171277:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009d5b7649/0xd5401cbabc0219a9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x82c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f242e expref: 223 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.171289:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.171298:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000038ed2a55/0xd5401cbabc021939 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x829:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2404 expref: 223 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.171299:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000efe92229/0xd5401cbabc02179c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x827:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23e8 expref: 223 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.171304:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009d5b7649/0xd5401cbabc0219a9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x82c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f242e expref: 223 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.171331:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000038ed2a55/0xd5401cbabc021939 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x829:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2404 expref: 223 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.171335:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000efe92229/0xd5401cbabc02179c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x827:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23e8 expref: 223 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.171340:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009d5b7649/0xd5401cbabc0219a9 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x82c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f242e expref: 223 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.171407:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000efe92229/0xd5401cbabc02179c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x827:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23e8 expref: 222 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.171417:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x827:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.171419:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x827:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.171423:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026720 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x827:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.171429:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026720 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x827:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.171431:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x829:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.171438:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x829:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.171447:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.171450:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02674a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x829:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.171462:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02674a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x829:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.171462:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x82c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.171470:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x82c:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.171475:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517208000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+220:2387:x1870153002467840:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 210us (779us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.171481:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02673c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x82c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.171485:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36450 00010000:00010000:0.0:1783520936.171507:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02673c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x82c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.171513:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.171519:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36451 00000100:00100000:2.0:1783520936.171531:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36452 00000100:00100000:1.0:1783520936.171553:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515477b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+219:2387:x1870153002468096:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.171553:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.171554:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515474380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+219:2387:x1870153002468608:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520936.171556:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94361b4a6450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+219:2387:x1870153002467968:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 306us (889us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.171583:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.171584:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36448 00000100:00100000:0.0:1783520936.171587:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351720b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+219:2387:x1870153002468352:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 334us (908us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.171589:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.171594:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ecd8f91e/0xd5401cbabc02198d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x82a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2412 expref: 219 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.171612:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000624178c1/0xd5401cbabc02184b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x828:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23f6 expref: 218 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.171623:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ecd8f91e/0xd5401cbabc02198d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x82a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2412 expref: 218 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.171623:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36449 00010000:00010000:0.0:1783520936.171657:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026751 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x82e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.171688:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ecd8f91e/0xd5401cbabc02198d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x82a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2412 expref: 217 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.171709:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026751 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x82e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.171712:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000624178c1/0xd5401cbabc02184b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x828:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23f6 expref: 216 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.171713:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x82a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.171731:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x82a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.171749:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc02672e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x82a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.171761:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc02672e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x82a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.171762:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026751 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x82e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.171770:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000624178c1/0xd5401cbabc02184b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x828:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f23f6 expref: 216 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.171786:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02674a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x829:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.171800:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.171819:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026751 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x82e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.171821:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02674a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x829:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.171826:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x828:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.171827:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515474380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+215:2387:x1870153002468608:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 286us (1074us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.171846:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x828:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.171863:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36452 00010000:00010000:1.0:1783520936.171870:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026727 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x828:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.171871:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02674a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x829:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.171879:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.171903:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.171913:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026727 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x828:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.171915:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026751 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x82e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.171916:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02674a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x829:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.171959:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.171966:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026751 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x82e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.171973:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026751 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x82e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.171976:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.171986:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026751 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x82e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.171989:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.171992:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.172001:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02674a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x829:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.172001:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026751 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x82e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.172017:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026727 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x828:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.172029:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbaa00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.172034:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026727 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x828:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.172047:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515477b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+214:2387:x1870153002468096:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 511us (1323us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.172054:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.172059:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.172063:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.172075:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026727 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x828:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.172076:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36451 00010000:00010000:2.0:1783520936.172090:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026727 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x828:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.172110:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.172118:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02674a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x829:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.172129:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02674a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x829:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.172134:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026720 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x827:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.172148:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02674a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x829:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.172152:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026720 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x827:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.172193:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc02674a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x829:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.172204:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026720 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x827:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.172218:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026720 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x827:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.172241:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+789:3628:x1870152993252096:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5101us (5241us total) trans 4294976356 rc 0/0 00000100:00100000:2.0:1783520936.172249:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514fdc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+789:3625:x1870152993250816:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6790us (8051us total) trans 4294976357 rc 0/0 00010000:00010000:1.0:1783520936.172254:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520936.172262:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00000100:00100000:0.0:1783520936.172266:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44861 00010000:00010000:3.0:1783520936.172269:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026727 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x828:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.172275:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44856 00010000:00010000:3.0:1783520936.172284:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.172289:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026727 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x828:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.172296:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026727 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x828:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.172310:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026727 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x828:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.172318:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026727 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x828:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.172319:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435121a7800/2, rc 0, transno 4294976356 00000004:00080000:2.0:1783520936.172324:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514fdf100/2, rc 0, transno 4294976357 00000100:00100000:0.0:1783520936.172340:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993252096:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.172343:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514fdf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993250816:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.172347:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.172352:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026720 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x827:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.172399:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.172404:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026720 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x827:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.172502:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507938700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+788:3625:x1870152993250688:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7677us (8530us total) trans 4294976358 rc 0/0 00000100:00100000:0.0:1783520936.172503:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993252864:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.172522:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44854 00000100:00100000:0.0:1783520936.172565:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.172575:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026743 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x82d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.172601:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:1.0:1783520936.172616:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026743 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x82d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.172618:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026720 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x827:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.172622:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993252736:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.172667:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.172670:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026743 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x82d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.172677:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026720 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x827:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.172683:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026743 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x82d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.172706:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026720 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x827:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.172721:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993252864 00000004:00080000:0.0:1783520936.172726:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350793a680/2, rc 0, transno 4294976358 00010000:00010000:1.0:1783520936.172727:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.172728:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993252864 00010000:00010000:3.0:1783520936.172739:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520936.172743:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350793a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993250688:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.172746:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026743 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x82d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.172757:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.172762:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026743 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x82d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.172763:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.172774:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993252736 00000100:00100000:2.0:1783520936.172779:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993252736 00010000:00010000:3.0:1783520936.172781:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026743 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x82d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.172802:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026743 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x82d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.172810:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026743 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x82d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.172816:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44862 00000100:00100000:0.0:1783520936.172817:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1783520936.172835:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+790:3626:x1870152993252864:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.172836:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993252992:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.172840:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.172846:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.172852:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.172854:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44863 00002000:00080000:2.0:1783520936.172865:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2096 count 1 00010000:00010000:3.0:1783520936.172866:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.172873:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+790:3626:x1870152993252736:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.172878:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.172887:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.172892:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026758 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x830:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.172895:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc02672e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x82a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.172907:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2095 count 1 00010000:00010000:3.0:1783520936.172923:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc02672e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x82a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.172927:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993252992 00010000:00010000:2.0:1783520936.172931:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000099566ab9/0xd5401cbabc0219e8 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x830:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2466 expref: 213 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.172933:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02675f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x82f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.172935:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993252992 00010000:00010000:3.0:1783520936.172938:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc02672e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x82a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.172970:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44864 00010000:00010000:3.0:1783520936.172982:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc02672e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x82a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.172983:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000268f9d6f/0xd5401cbabc0219d3 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x82f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2458 expref: 213 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.172985:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507939880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+791:3628:x1870152993252992:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.173018:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2097 count 1 00010000:00010000:1.0:1783520936.173038:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x82f:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.173042:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026766 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x831:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.173044:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x830:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.173060:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x82f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.173061:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x830:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.173085:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026758 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x830:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.173085:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02675f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x82f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.173089:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.173095:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000fb3d5bb6/0xd5401cbabc0219ef lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x831:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2474 expref: 213 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.173143:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x831:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.173148:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x831:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.173152:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026766 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x831:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.173157:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000268f9d6f/0xd5401cbabc0219d3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x82f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2458 expref: 213 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.173172:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000099566ab9/0xd5401cbabc0219e8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x830:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2466 expref: 213 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.173198:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.173203:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000268f9d6f/0xd5401cbabc0219d3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x82f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2458 expref: 213 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.173204:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000099566ab9/0xd5401cbabc0219e8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x830:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2466 expref: 213 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.173205:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc02672e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x82a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.173214:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+792:3628:x1870152993251200:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7474us (7612us total) trans 4294976360 rc 0/0 00000100:00100000:3.0:1783520936.173256:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44857 00010000:00010000:0.0:1783520936.173262:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.173269:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc02672e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x82a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.173270:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a5f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993253120:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.173275:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515477800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993253248:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.173296:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc02672e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x82a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.173308:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc02672e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x82a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.173316:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc02672e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x82a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.173343:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.173351:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.173355:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.173402:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514825600 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.173416:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514fdd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+792:3627:x1870152993250944:12345-0@lo:6:osp-syn-1-0.0 Request processed in 8065us (8945us total) trans 4294976361 rc 0/0 00000100:00100000:3.0:1783520936.173427:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44855 00000004:00080000:0.0:1783520936.173429:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363fe56d80/2, rc 0, transno 4294976360 00000100:00100000:0.0:1783520936.173446:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363fe56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993251200:0@lo:6:osp-syn-1-0.0 00000004:00080000:1.0:1783520936.173470:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514fded80/2, rc 0, transno 4294976361 00000100:00100000:1.0:1783520936.173479:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514fded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993250944:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.173509:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1783520936.173520:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351510aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993253376:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.173550:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b06680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993253504:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.173567:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.173586:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000fb3d5bb6/0xd5401cbabc0219ef lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x831:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2474 expref: 213 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.173602:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000fb3d5bb6/0xd5401cbabc0219ef lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x831:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2474 expref: 213 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.173616:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993253376 00000100:00100000:2.0:1783520936.173618:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.173623:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470cc0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.173625:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993253376 00000100:00100000:0.0:1783520936.173631:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b06d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993253632:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.173666:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993253504 00000100:00100000:3.0:1783520936.173670:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993253504 00010000:00010000:1.0:1783520936.173679:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02673c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x82c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.173691:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44865 00000100:00100000:0.0:1783520936.173691:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff9436298a6480 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.173712:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02673c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x82c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.173713:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+793:3627:x1870152993253376:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.173726:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44866 00002000:00080000:2.0:1783520936.173738:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2098 count 1 00010000:00010000:1.0:1783520936.173739:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02673c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x82c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.173742:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515476a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+793:3625:x1870152993253504:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.173759:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02676d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x832:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.173770:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2099 count 1 00010000:00010000:0.0:1783520936.173777:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026735 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x82b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.173778:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02673c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x82c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.173790:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026774 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x833:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.173802:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005aa0c211/0xd5401cbabc0219fd lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x832:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2482 expref: 213 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.173832:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026735 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x82b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.173846:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.173858:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026735 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x82b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.173861:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x832:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.173871:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x832:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.173873:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026735 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x82b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.173877:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02676d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x832:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.173892:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520936.173899:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000079ab92c0/0xd5401cbabc021a0b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x833:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2490 expref: 213 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.173908:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005aa0c211/0xd5401cbabc0219fd lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x832:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2482 expref: 213 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.173910:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x833:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.173917:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x833:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.173924:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026774 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x833:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.173930:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005aa0c211/0xd5401cbabc0219fd lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x832:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2482 expref: 213 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.173945:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515474000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993253760:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.173948:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1783520936.173957:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02673c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x82c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.173966:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.173971:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02673c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x82c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.173981:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02673c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x82c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.173986:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435122b6c00 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.173997:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+795:3626:x1870152993251584:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7358us (7450us total) trans 4294976362 rc 0/0 00010000:00010000:3.0:1783520936.174002:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02673c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x82c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.174008:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+795:3626:x1870152993251456:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7589us (7743us total) trans 4294976363 rc 0/0 00000100:00100000:1.0:1783520936.174014:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44859 00010000:00010000:3.0:1783520936.174031:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02673c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x82c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.174036:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44858 00010000:00010000:3.0:1783520936.174075:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.174079:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026735 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x82b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.174084:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435121a4e00/2, rc 0, transno 4294976362 00010000:00010000:3.0:1783520936.174087:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.174094:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026735 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x82b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.174104:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026735 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x82b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.174108:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993251584:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.174139:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026735 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x82b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.174144:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+793:3628:x1870152993251840:12345-0@lo:6:osp-syn-1-0.0 Request processed in 7255us (7351us total) trans 4294976359 rc 0/0 00010000:00010000:3.0:1783520936.174158:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026735 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x82b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.174160:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44860 00010000:00010000:3.0:1783520936.174183:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.174190:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.174197:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.174202:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000004:00080000:1.0:1783520936.174209:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435121a4a80/2, rc 0, transno 4294976363 00000004:00080000:0.0:1783520936.174213:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435121a5c00/2, rc 0, transno 4294976359 00000100:00100000:2.0:1783520936.174219:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514fdf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993253888:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.174230:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993251456:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.174231:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993251840:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.174255:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.174271:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993253888 00000100:00100000:0.0:1783520936.174275:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993253888 00000100:00100000:2.0:1783520936.174305:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44867 00000100:00100000:2.0:1783520936.174320:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635739850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+793:3625:x1870152993253888:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.174338:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2100 count 1 00010000:00010000:2.0:1783520936.174353:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02677b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x834:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.174408:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516682a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993254016:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.174409:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b1c200d5/0xd5401cbabc021a12 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x834:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f249e expref: 213 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.174436:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x834:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.174440:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x834:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.174443:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02677b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x834:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.174470:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.174477:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516682300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993254144:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.174479:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b1c200d5/0xd5401cbabc021a12 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x834:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f249e expref: 213 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.174485:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.174511:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b1c200d5/0xd5401cbabc021a12 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x834:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f249e expref: 213 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.174526:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515474700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993254272:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.174532:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000079ab92c0/0xd5401cbabc021a0b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x833:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2490 expref: 213 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.174544:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.174546:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993254016 00010000:00010000:3.0:1783520936.174554:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000079ab92c0/0xd5401cbabc021a0b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x833:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2490 expref: 213 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.174555:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993254016 00000100:00100000:3.0:1783520936.174576:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503371500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993254400:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.174579:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a87a80 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.174610:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993254144 00000100:00100000:1.0:1783520936.174614:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993254144 00000100:00100000:3.0:1783520936.174629:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6f00 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.174635:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44868 00000100:00100000:0.0:1783520936.174656:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516681f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+795:3627:x1870152993254016:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.174663:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44869 00002000:00080000:0.0:1783520936.174673:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2101 count 1 00000100:00100000:1.0:1783520936.174677:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b04a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+795:3628:x1870152993254144:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.174693:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026782 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x835:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.174701:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2102 count 1 00010000:00010000:0.0:1783520936.174709:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000233dfc57/0xd5401cbabc021a27 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x835:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f24ac expref: 213 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.174719:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x835:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.174723:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x835:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.174724:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026789 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x836:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.174731:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026782 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x835:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.174761:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000cc6b3816/0xd5401cbabc022a03 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x836:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f30f3 expref: 213 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.174769:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x836:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.174772:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x836:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.174775:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026789 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x836:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.174783:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000233dfc57/0xd5401cbabc021a27 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x835:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24ac expref: 213 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.174796:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000233dfc57/0xd5401cbabc021a27 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x835:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24ac expref: 213 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.174800:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000cc6b3816/0xd5401cbabc022a03 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x836:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30f3 expref: 213 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.174846:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000cc6b3816/0xd5401cbabc022a03 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x836:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30f3 expref: 213 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.174847:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b07100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993254528:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.174878:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9ea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993254656:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.174900:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895840 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.174913:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470480 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.175236:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515477800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993253248:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.175266:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026758 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x830:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.175419:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a5f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993253120:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.175444:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02675f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x82f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.175575:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.175598:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002468864 00000100:00100000:3.0:1783520936.175603:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002468864 00010000:00010000:3.0:1783520936.175624:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000099566ab9/0xd5401cbabc0219e8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x830:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2466 expref: 214 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.175685:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36453 00000100:00100000:0.0:1783520936.175698:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503372a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+214:2388:x1870153002468864:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.175708:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.175714:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000099566ab9/0xd5401cbabc0219e8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x830:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2466 expref: 214 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.175726:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000099566ab9/0xd5401cbabc0219e8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x830:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2466 expref: 214 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.175742:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000099566ab9/0xd5401cbabc0219e8 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x830:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2466 expref: 214 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.175754:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x830:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.175757:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x830:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.175762:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026758 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x830:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.175770:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026758 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x830:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.175794:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.175828:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503372a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+213:2388:x1870153002468864:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 132us (254us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.175829:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026758 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x830:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.175844:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36453 00010000:00010000:2.0:1783520936.175847:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026758 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x830:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.175862:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026758 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x830:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.175871:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026758 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x830:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.175891:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.175910:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.175918:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026758 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x830:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.175927:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.175931:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026758 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x830:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.175942:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026758 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x830:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.175955:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026758 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x830:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.175962:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026758 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x830:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.176014:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+796:3626:x1870152993252864:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3187us (3453us total) trans 4294976364 rc 0/0 00000100:00100000:2.0:1783520936.176023:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44862 00010000:00010000:0.0:1783520936.176042:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.176046:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.176051:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520936.176066:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943508ee7100/2, rc 0, transno 4294976364 00000100:00100000:1.0:1783520936.176086:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993252864:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.176141:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1783520936.176156:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993254784:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.176197:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.176254:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993254784 00000100:00100000:0.0:1783520936.176261:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993254784 00000100:00100000:0.0:1783520936.176300:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44870 00000100:00100000:0.0:1783520936.176308:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+796:3626:x1870152993254784:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.176324:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2103 count 1 00010000:00010000:0.0:1783520936.176352:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026790 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x837:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.176423:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000066a91378/0xd5401cbabc021ac1 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x837:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f24c8 expref: 212 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.176432:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x837:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.176437:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x837:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.176442:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026790 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x837:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.176468:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000066a91378/0xd5401cbabc021ac1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x837:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24c8 expref: 212 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.176479:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000066a91378/0xd5401cbabc021ac1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x837:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24c8 expref: 212 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.176504:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b04e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993254912:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.176555:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514219780 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.177136:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515474000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993253760:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.177155:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b06d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993253632:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.177157:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02676d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x832:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.177179:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026766 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x831:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.177197:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515474700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993254272:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.177210:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02677b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x834:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.177257:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.177282:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002468992 00000100:00100000:2.0:1783520936.177287:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002468992 00000100:00100000:0.0:1783520936.177298:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.177307:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000268f9d6f/0xd5401cbabc0219d3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x82f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2458 expref: 213 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.177331:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002469248 00000100:00100000:2.0:1783520936.177335:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002469248 00000100:00100000:0.0:1783520936.177339:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.177349:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000fb3d5bb6/0xd5401cbabc0219ef lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x831:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2474 expref: 214 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.177381:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503371500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993254400:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.177396:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002469120 00010000:00010000:1.0:1783520936.177398:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026774 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x833:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.177401:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002469120 00000100:00100000:0.0:1783520936.177413:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.177420:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005aa0c211/0xd5401cbabc0219fd lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x832:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2482 expref: 215 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.177432:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002469376 00000100:00100000:1.0:1783520936.177438:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002469376 00000100:00100000:3.0:1783520936.177443:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36454 00000100:00100000:0.0:1783520936.177444:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.177454:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36455 00000100:00100000:3.0:1783520936.177460:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b06d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+216:2389:x1870153002468992:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.177462:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b1c200d5/0xd5401cbabc021a12 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x834:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f249e expref: 216 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.177470:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94361b4a5c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+216:2389:x1870153002469248:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.177472:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.177486:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.177505:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002469504 00010000:00010000:2.0:1783520936.177508:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000fb3d5bb6/0xd5401cbabc0219ef lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x831:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2474 expref: 216 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.177510:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002469504 00010000:00010000:2.0:1783520936.177524:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000fb3d5bb6/0xd5401cbabc0219ef lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x831:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2474 expref: 217 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.177530:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000079ab92c0/0xd5401cbabc021a0b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x833:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2490 expref: 217 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.177530:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.177552:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.177554:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002469632 00000100:00100000:1.0:1783520936.177557:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002469632 00010000:00010000:2.0:1783520936.177559:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000fb3d5bb6/0xd5401cbabc0219ef lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x831:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2474 expref: 217 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.177570:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000233dfc57/0xd5401cbabc021a27 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x835:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24ac expref: 217 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.177574:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x831:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.177580:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x831:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.177582:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002469760 00000100:00100000:1.0:1783520936.177585:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002469760 00010000:00010000:2.0:1783520936.177586:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026766 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x831:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.177596:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026766 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x831:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.177598:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000cc6b3816/0xd5401cbabc022a03 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x836:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30f3 expref: 218 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.177616:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000268f9d6f/0xd5401cbabc0219d3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x82f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2458 expref: 216 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.177622:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b07100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993254528:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.177632:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.177646:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36456 00010000:00010000:3.0:1783520936.177647:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000268f9d6f/0xd5401cbabc0219d3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x82f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2458 expref: 218 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.177647:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026782 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x835:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.177656:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b05c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+218:2388:x1870153002469120:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.177668:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94361b4a5c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+218:2389:x1870153002469248:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 201us (367us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.177679:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36455 00010000:00010000:3.0:1783520936.177684:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000268f9d6f/0xd5401cbabc0219d3 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x82f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2458 expref: 218 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.177685:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026766 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x831:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.177689:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.177700:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005aa0c211/0xd5401cbabc0219fd lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x832:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2482 expref: 217 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.177703:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36457 00010000:00010000:0.0:1783520936.177712:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026766 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x831:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.177716:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x82f:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.177718:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b07480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+216:2388:x1870153002469376:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.177726:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x82f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.177734:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.177736:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02675f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x82f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.177738:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026766 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x831:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.177739:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005aa0c211/0xd5401cbabc0219fd lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x832:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2482 expref: 216 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.177743:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b1c200d5/0xd5401cbabc021a12 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x834:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f249e expref: 216 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.177764:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02675f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x82f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.177767:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026766 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x831:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.177778:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b1c200d5/0xd5401cbabc021a12 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x834:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f249e expref: 216 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.177781:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005aa0c211/0xd5401cbabc0219fd lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x832:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2482 expref: 216 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.177789:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.177798:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b1c200d5/0xd5401cbabc021a12 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x834:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f249e expref: 216 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.177802:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x832:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.177803:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.177811:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x832:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.177811:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026766 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x831:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.177815:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x834:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.177825:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x834:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.177828:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02676d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x832:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.177833:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.177835:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02677b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x834:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.177841:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.177841:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026766 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x831:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.177853:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02676d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x832:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.177863:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b06d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+214:2389:x1870153002468992:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 408us (605us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.177864:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02677b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x834:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.177866:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026766 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x831:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.177880:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36454 00010000:00010000:1.0:1783520936.177891:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.177899:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026766 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x831:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.177909:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026766 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x831:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.177912:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.177917:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b05c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+213:2388:x1870153002469120:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 263us (589us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.177925:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36458 00000100:00100000:1.0:1783520936.177933:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36456 00000100:00100000:3.0:1783520936.177934:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b07b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+213:2390:x1870153002469504:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.177941:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.177942:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b07480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+213:2388:x1870153002469376:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 224us (528us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.177946:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.177946:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.177952:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.177955:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000079ab92c0/0xd5401cbabc021a0b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x833:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2490 expref: 212 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.177956:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36459 00010000:00010000:3.0:1783520936.177975:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000079ab92c0/0xd5401cbabc021a0b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x833:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2490 expref: 212 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.177976:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36457 00000100:00100000:1.0:1783520936.177977:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517244e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+212:2388:x1870153002469632:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.177992:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000079ab92c0/0xd5401cbabc021a0b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x833:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2490 expref: 211 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.177994:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.178003:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36460 00010000:00010000:1.0:1783520936.178004:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000233dfc57/0xd5401cbabc021a27 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x835:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24ac expref: 211 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.178017:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517247100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+211:2388:x1870153002469760:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.178033:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x833:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.178034:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.178035:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000233dfc57/0xd5401cbabc021a27 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x835:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24ac expref: 210 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.178041:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x833:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.178044:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000cc6b3816/0xd5401cbabc022a03 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x836:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30f3 expref: 210 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.178052:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026774 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x833:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.178066:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000233dfc57/0xd5401cbabc021a27 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x835:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24ac expref: 210 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.178069:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000cc6b3816/0xd5401cbabc022a03 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x836:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30f3 expref: 210 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.178070:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507939880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+797:3628:x1870152993252992:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5084us (5188us total) trans 4294976365 rc 0/0 00000100:00100000:0.0:1783520936.178089:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44864 00010000:00010000:1.0:1783520936.178097:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x835:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.178101:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x836:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.178101:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x835:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.178110:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x836:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.178111:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026782 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x835:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.178117:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026789 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x836:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.178121:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026782 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x835:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.178129:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943508ee5500/2, rc 0, transno 4294976365 00010000:00010000:2.0:1783520936.178133:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026789 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x836:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.178146:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993252992:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.178152:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.178152:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.178177:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517247100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+209:2388:x1870153002469760:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 164us (626us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.178181:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026774 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x833:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.178183:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517244e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+209:2388:x1870153002469632:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 215us (655us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.178204:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36460 00000100:00100000:0.0:1783520936.178215:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1783520936.178216:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36459 00010000:00010000:3.0:1783520936.178219:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.178231:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517245180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993255040:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.178240:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02676d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x832:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.178246:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b07b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+207:2390:x1870153002469504:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 313us (800us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.178254:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02676d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x832:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.178259:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026774 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x833:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.178271:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.178274:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36458 00010000:00010000:2.0:1783520936.178281:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02676d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x832:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.178286:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026774 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x833:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.178300:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02676d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x832:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.178302:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026774 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x833:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.178311:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993255040 00000100:00100000:0.0:1783520936.178316:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993255040 00010000:00010000:1.0:1783520936.178321:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026774 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x833:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.178334:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.178334:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520936.178342:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:0.0:1783520936.178344:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44871 00000100:00100000:0.0:1783520936.178351:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b06d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+797:3625:x1870152993255040:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.178390:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2104 count 1 00010000:00010000:0.0:1783520936.178404:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026797 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x838:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.178414:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007a5a4b8a/0xd5401cbabc021b15 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x838:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f24d6 expref: 206 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.178419:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520936.178420:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x838:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.178425:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x838:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.178426:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02676d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x832:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.178430:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026797 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x838:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.178441:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.178445:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02676d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x832:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.178449:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515476a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+799:3625:x1870152993253504:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4710us (4837us total) trans 4294976366 rc 0/0 00010000:00010000:0.0:1783520936.178455:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007a5a4b8a/0xd5401cbabc021b15 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x838:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24d6 expref: 206 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.178458:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02676d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x832:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.178463:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44866 00010000:00010000:0.0:1783520936.178478:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007a5a4b8a/0xd5401cbabc021b15 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x838:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24d6 expref: 206 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.178482:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02676d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x832:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.178503:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+798:3627:x1870152993253376:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4778us (4921us total) trans 4294976367 rc 0/0 00000100:00100000:0.0:1783520936.178503:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b07100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993255168:192.168.204.33@tcp:104: 00000004:00080000:1.0:1783520936.178509:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513b06680/2, rc 0, transno 4294976366 00010000:00010000:3.0:1783520936.178521:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc02676d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x832:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.178523:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b06680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993253504:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.178524:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44865 00000100:00100000:0.0:1783520936.178553:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895840 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.178558:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002469888 00010000:00010000:3.0:1783520936.178561:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:2.0:1783520936.178563:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002469888 00010000:00010000:3.0:1783520936.178566:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026774 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x833:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.178571:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993255296:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.178574:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026782 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x835:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.178584:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000066a91378/0xd5401cbabc021ac1 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x837:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24c8 expref: 207 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.178598:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.178599:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026782 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x835:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.178604:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026774 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x833:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.178608:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.178623:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36461 00010000:00010000:0.0:1783520936.178626:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026782 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x835:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.178629:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026774 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x833:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.178635:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503372a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+207:2387:x1870153002469888:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000004:00080000:1.0:1783520936.178640:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351510aa00/2, rc 0, transno 4294976367 00010000:00010000:0.0:1783520936.178650:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026782 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x835:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.178651:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.178652:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351510aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993253376:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.178656:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026774 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x833:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.178662:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000066a91378/0xd5401cbabc021ac1 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x837:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24c8 expref: 207 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.178684:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.178685:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026774 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x833:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.178690:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000066a91378/0xd5401cbabc021ac1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x837:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24c8 expref: 207 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.178698:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520936.178699:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:0.0:1783520936.178708:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026782 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x835:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.178712:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993255424:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.178716:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.178717:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x837:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.178719:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.178723:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.178727:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x837:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.178728:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.178732:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.178734:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026790 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x837:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.178735:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026782 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x835:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.178750:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.178757:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026790 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x837:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.178760:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026782 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x835:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.178785:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.178786:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026782 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x835:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.178796:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc026782 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x835:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.178805:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993255296 00000100:00100000:1.0:1783520936.178811:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993255296 00000100:00100000:2.0:1783520936.178813:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503372a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+207:2387:x1870153002469888:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 179us (480us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.178821:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.178822:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36461 00010000:00010000:0.0:1783520936.178824:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.178829:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.178841:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993255424 00000100:00100000:1.0:1783520936.178845:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993255424 00000100:00100000:2.0:1783520936.178855:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44872 00000100:00100000:2.0:1783520936.178868:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+799:3627:x1870152993255296:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.178874:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44873 00010000:00010000:3.0:1783520936.178882:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02675f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x82f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.178882:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2105 count 1 00000100:00100000:1.0:1783520936.178887:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+800:3627:x1870152993255424:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.178900:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02679e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x839:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.178901:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02675f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x82f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.178909:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2106 count 1 00000100:00100000:0.0:1783520936.178928:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516681f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+800:3627:x1870152993254016:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4280us (4456us total) trans 4294976368 rc 0/0 00010000:00010000:1.0:1783520936.178929:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0267a5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x83a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.178932:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005ef48638/0xd5401cbabc021b70 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x839:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f24e4 expref: 206 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.178947:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44868 00010000:00010000:2.0:1783520936.178963:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x839:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.178967:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000cba9e215/0xd5401cbabc021baf lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x83a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f24f2 expref: 206 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.178968:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x839:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.178973:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02675f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x82f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.178975:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02679e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x839:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.178978:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516682a00/2, rc 0, transno 4294976368 00000100:00100000:0.0:1783520936.178989:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516682a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993254016:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.178991:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x83a:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.179001:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02675f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x82f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.179002:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x83a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.179010:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0267a5 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x83a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.179019:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005ef48638/0xd5401cbabc021b70 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x839:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24e4 expref: 206 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.179024:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.179032:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.179035:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005ef48638/0xd5401cbabc021b70 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x839:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24e4 expref: 206 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.179039:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02675f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x82f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.179040:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993255552:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.179061:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb400 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520936.179063:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515474700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993255680:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.179067:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02675f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x82f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.179078:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02675f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x82f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.179085:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.179099:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02675f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x82f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.179105:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02675f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x82f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.179106:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514825600 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.179113:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+800:3626:x1870152993252736:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6246us (6453us total) trans 4294976369 rc 0/0 00000100:00100000:3.0:1783520936.179129:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44863 00010000:00010000:2.0:1783520936.179132:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02677b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x834:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.179140:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.179143:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02677b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x834:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.179143:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943508ee4e00/2, rc 0, transno 4294976369 00010000:00010000:0.0:1783520936.179143:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.179148:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993255552 00010000:00010000:0.0:1783520936.179149:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.179154:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993255552 00010000:00010000:2.0:1783520936.179160:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02677b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x834:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.179160:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993252736:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.179171:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02677b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x834:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.179184:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44874 00000100:00100000:3.0:1783520936.179194:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+800:3625:x1870152993255552:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.179199:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.179209:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520936.179212:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993255808:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.179216:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02677b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x834:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.179225:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.179229:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02677b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x834:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.179240:0:9151:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000066a91378/0xd5401cbabc021ac1 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x837:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24c8 expref: 206 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.179246:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02677b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x834:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.179252:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.179264:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b04e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993254912:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.179287:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026790 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x837:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.179288:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635739850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+801:3625:x1870152993253888:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4973us (5036us total) trans 4294976370 rc 0/0 00000004:00080000:1.0:1783520936.179302:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514fdf480/2, rc 0, transno 4294976370 00000100:00100000:2.0:1783520936.179307:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44867 00010000:00010000:0.0:1783520936.179313:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026790 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x837:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.179317:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514fdf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993253888:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.179320:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02677b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x834:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.179331:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026790 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x837:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.179336:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02677b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x834:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.179349:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026790 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x837:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.179394:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026790 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x837:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.179397:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.179401:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.179407:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.179417:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00002000:00080000:3.0:1783520936.179424:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2107 count 1 00000100:00100000:1.0:1783520936.179430:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993255936:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.179433:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993255808 00000100:00100000:0.0:1783520936.179438:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993255808 00010000:00010000:3.0:1783520936.179442:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0267ac lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x83b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.179456:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000074ec2bc/0xd5401cbabc021c1f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x83b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2500 expref: 205 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.179463:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x83b:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.179467:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x83b:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.179471:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.179473:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44875 00010000:00010000:3.0:1783520936.179475:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0267ac lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x83b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.179487:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993255936 00000100:00100000:2.0:1783520936.179500:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+801:3626:x1870152993255808:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.179502:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993255936 00002000:00080000:2.0:1783520936.179517:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2108 count 1 00010000:00010000:3.0:1783520936.179518:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000074ec2bc/0xd5401cbabc021c1f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x83b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2500 expref: 205 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.179534:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0267b3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x83c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.179537:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44876 00010000:00010000:1.0:1783520936.179538:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000cba9e215/0xd5401cbabc021baf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x83a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24f2 expref: 205 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.179551:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+802:3628:x1870152993255936:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.179559:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f4202631/0xd5401cbabc021c7a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x83c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f250e expref: 205 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.179567:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x83c:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.179568:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000cba9e215/0xd5401cbabc021baf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x83a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24f2 expref: 205 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.179570:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2109 count 1 00010000:00010000:2.0:1783520936.179571:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x83c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.179579:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0267b3 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x83c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.179587:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0267ba lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x83d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.179594:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9e680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993256192:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.179607:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f4202631/0xd5401cbabc021c7a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x83c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f250e expref: 205 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.179610:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003f1b3eb0/0xd5401cbabc021cc7 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x83d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f251c expref: 205 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.179618:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f4202631/0xd5401cbabc021c7a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x83c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f250e expref: 205 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.179633:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x83d:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.179643:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x83d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.179654:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0267ba lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x83d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.179663:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000074ec2bc/0xd5401cbabc021c1f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x83b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2500 expref: 205 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.179667:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515476a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993256320:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.179687:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503370380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993256064:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.179694:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003f1b3eb0/0xd5401cbabc021cc7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x83d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f251c expref: 205 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.179706:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003f1b3eb0/0xd5401cbabc021cc7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x83d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f251c expref: 205 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.179717:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993256448:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.179728:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943508fa6600 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.179728:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a87a80 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.179731:0:8410:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cc6b3816/0xd5401cbabc022a03 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x836:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30f3 expref: 205 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.179743:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff9436298a6480 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.179748:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9ea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993254656:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.179769:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026789 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x836:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.179770:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.179780:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.179782:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026789 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x836:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.179787:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026790 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x837:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.179794:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026789 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x836:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.179802:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbebe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.179809:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026790 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x837:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.179810:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026789 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x836:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.179829:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026790 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x837:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.179832:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026789 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x836:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.179845:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+803:3626:x1870152993254784:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3535us (3645us total) trans 4294976371 rc 0/0 00010000:00010000:3.0:1783520936.179846:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026790 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x837:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.179864:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e9fb80/2, rc 0, transno 4294976371 00010000:00010000:3.0:1783520936.179869:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026790 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x837:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.179871:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44870 00000100:00100000:1.0:1783520936.179876:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993254784:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.179897:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.179900:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.179902:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.179935:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993256576:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.179964:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.179980:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993256576 00000100:00100000:3.0:1783520936.179985:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993256576 00000100:00100000:1.0:1783520936.180019:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124700c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.180021:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44877 00000100:00100000:3.0:1783520936.180031:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+803:3628:x1870152993256576:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.180036:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00002000:00080000:3.0:1783520936.180044:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2110 count 1 00010000:00010000:0.0:1783520936.180051:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.180058:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026789 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x836:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.180060:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0267c1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x83e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.180078:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.180084:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026789 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x836:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.180085:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b4c4261c/0xd5401cbabc0228f2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x83e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fe2 expref: 204 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.180109:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x83e:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.180109:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026789 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x836:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.180117:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x83e:0x0].0x0 since=0s v=(1 0) 00000100:00100000:1.0:1783520936.180117:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b04a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+804:3628:x1870152993254144:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5445us (5579us total) trans 4294976372 rc 0/0 00010000:00010000:3.0:1783520936.180128:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0267c1 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x83e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.180132:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44869 00010000:00010000:0.0:1783520936.180133:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026789 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x836:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.180149:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026789 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x836:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.180162:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b4c4261c/0xd5401cbabc0228f2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x83e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fe2 expref: 204 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.180164:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516682300/2, rc 0, transno 4294976372 00010000:00010000:0.0:1783520936.180167:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.180170:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.180173:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.180178:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516682300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993254144:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.180182:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b4c4261c/0xd5401cbabc0228f2 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x83e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fe2 expref: 204 pid: 12734 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.180195:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503370e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993256704:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.180220:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993256832:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.180242:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57d80 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.180248:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.180283:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993256832 00000100:00100000:0.0:1783520936.180288:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993256832 00000100:00100000:0.0:1783520936.180313:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44878 00000100:00100000:0.0:1783520936.180317:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+804:3625:x1870152993256832:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.180326:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2111 count 1 00010000:00010000:0.0:1783520936.180334:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0267c8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x83f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.180342:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007da35a1b/0xd5401cbabc021d7d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x83f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2538 expref: 204 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.180346:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x83f:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.180349:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x83f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.180351:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0267c8 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x83f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.180385:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007da35a1b/0xd5401cbabc021d7d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x83f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2538 expref: 204 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.180391:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007da35a1b/0xd5401cbabc021d7d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x83f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2538 expref: 204 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.180399:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f0380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993256960:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.180425:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514219780 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.180568:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b07100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993255168:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.180579:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026797 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x838:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.180947:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.180989:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.181013:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002470016 00000100:00100000:2.0:1783520936.181017:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002470016 00000100:00100000:1.0:1783520936.181030:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002470144 00000100:00100000:1.0:1783520936.181036:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002470144 00010000:00010000:2.0:1783520936.181037:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007a5a4b8a/0xd5401cbabc021b15 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x838:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24d6 expref: 205 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.181058:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005ef48638/0xd5401cbabc021b70 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x839:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24e4 expref: 206 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.181065:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36462 00000100:00100000:2.0:1783520936.181077:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94361b4a6c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+206:2387:x1870153002470016:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.181086:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.181092:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007a5a4b8a/0xd5401cbabc021b15 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x838:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24d6 expref: 206 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.181098:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36463 00010000:00010000:2.0:1783520936.181112:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007a5a4b8a/0xd5401cbabc021b15 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x838:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24d6 expref: 206 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.181113:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515474000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+206:2390:x1870153002470144:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.181128:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.181135:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007a5a4b8a/0xd5401cbabc021b15 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x838:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24d6 expref: 206 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.181136:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005ef48638/0xd5401cbabc021b70 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x839:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24e4 expref: 206 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.181157:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x838:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.181163:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x838:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.181163:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005ef48638/0xd5401cbabc021b70 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x839:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24e4 expref: 205 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.181174:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026797 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x838:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.181180:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x839:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.181185:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026797 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x838:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.181185:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x839:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.181190:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02679e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x839:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.181203:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02679e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x839:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.181214:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.181238:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515474000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+205:2390:x1870153002470144:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 130us (248us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.181244:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36463 00010000:00010000:1.0:1783520936.181345:0:35274:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005ef48638/0xd5401cbabc021b70 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x839:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24e4 expref: 204 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.181356:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515474700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993255680:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.181393:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02679e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x839:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.181402:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02679e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x839:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.181414:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02679e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x839:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.181424:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02679e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x839:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.181431:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.181433:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02679e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x839:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.181441:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026797 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x838:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.181453:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.181456:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026797 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x838:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.181463:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026797 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x838:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.181470:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026797 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x838:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.181481:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.181486:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02679e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x839:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.181502:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94361b4a6c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+203:2387:x1870153002470016:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 416us (543us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.181516:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36462 00010000:00010000:0.0:1783520936.181516:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.181523:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02679e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x839:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.181533:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02679e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x839:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.181545:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02679e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x839:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.181552:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc02679e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x839:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.181556:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+805:3627:x1870152993255296:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2693us (2954us total) trans 4294976373 rc 0/0 00000100:00100000:1.0:1783520936.181564:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44872 00010000:00010000:0.0:1783520936.181573:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.181575:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.181577:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.181597:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000004:00080000:0.0:1783520936.181597:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e9ed80/2, rc 0, transno 4294976373 00000100:00100000:0.0:1783520936.181607:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993255296:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.181610:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.181617:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026797 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x838:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.181625:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.181629:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026797 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x838:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.181640:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026797 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x838:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.181654:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993257088:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.181655:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026797 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x838:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.181668:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026797 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x838:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.181678:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b06d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+805:3625:x1870152993255040:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3326us (3411us total) trans 4294976374 rc 0/0 00010000:00010000:3.0:1783520936.181685:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.181685:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.181686:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44871 00010000:00010000:3.0:1783520936.181688:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.181692:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.181703:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993257088 00000100:00100000:1.0:1783520936.181708:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993257088 00000004:00080000:0.0:1783520936.181725:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517245180/2, rc 0, transno 4294976374 00000100:00100000:0.0:1783520936.181733:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517245180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993255040:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.181736:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44879 00000100:00100000:1.0:1783520936.181743:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b05c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+805:3626:x1870152993257088:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.181756:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2112 count 1 00010000:00010000:1.0:1783520936.181768:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0267cf lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x840:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.181785:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e2608d10/0xd5401cbabc021ddf lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x840:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2546 expref: 202 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.181789:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:1.0:1783520936.181792:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x840:0x0].0x0 since=38s v=(0 0) 00000100:00100000:0.0:1783520936.181795:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993257216:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.181796:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x840:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.181800:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0267cf lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x840:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.181815:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.181817:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e2608d10/0xd5401cbabc021ddf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x840:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2546 expref: 202 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.181827:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e2608d10/0xd5401cbabc021ddf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x840:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2546 expref: 202 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.181838:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a6a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993257344:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.181841:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993257216 00000100:00100000:0.0:1783520936.181844:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993257216 00000100:00100000:0.0:1783520936.181866:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44880 00000100:00100000:0.0:1783520936.181872:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b07b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+806:3628:x1870152993257216:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.181874:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470000 going to sleep for 11 seconds 00002000:00080000:0.0:1783520936.181883:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2113 count 1 00010000:00010000:0.0:1783520936.181895:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc0267d6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x841:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.181907:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004e452570/0xd5401cbabc021e1e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x841:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2554 expref: 202 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.181914:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x841:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.181917:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x841:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.181921:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc0267d6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x841:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.181942:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004e452570/0xd5401cbabc021e1e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x841:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2554 expref: 202 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.181952:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004e452570/0xd5401cbabc021e1e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x841:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2554 expref: 202 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.181962:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b07100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993257472:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.181998:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895840 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.182341:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9e680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993256192:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.182354:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0267a5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x83a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.182444:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515476a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993256320:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.182462:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0267b3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x83c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.182556:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503370380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993256064:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.182575:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.182602:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002470400 00000100:00100000:0.0:1783520936.182610:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993256448:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.182616:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0267ac lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x83b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.182616:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002470400 00010000:00010000:0.0:1783520936.182631:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0267ba lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x83d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.182638:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f4202631/0xd5401cbabc021c7a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x83c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f250e expref: 203 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.182664:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f0380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993256960:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.182670:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36464 00010000:00010000:0.0:1783520936.182684:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0267c8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x83f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.182688:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b04e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+203:2387:x1870153002470400:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.182704:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.182711:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f4202631/0xd5401cbabc021c7a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x83c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f250e expref: 203 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.182723:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503370e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993256704:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.182724:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f4202631/0xd5401cbabc021c7a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x83c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f250e expref: 203 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.182743:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0267c1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x83e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.182750:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f4202631/0xd5401cbabc021c7a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x83c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f250e expref: 203 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.182780:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x83c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.182783:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x83c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.182788:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0267b3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x83c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.182796:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0267b3 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x83c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.182813:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.182850:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b04e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+202:2387:x1870153002470400:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 171us (276us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.182857:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36464 00010000:00010000:2.0:1783520936.182880:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0267b3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x83c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.182893:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0267b3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x83c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.182901:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0267b3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x83c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.182908:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0267b3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x83c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.182927:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.182955:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.182961:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0267b3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x83c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.182970:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.182973:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0267b3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x83c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.182982:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0267b3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x83c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.182995:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0267b3 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x83c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.183003:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0267b3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x83c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.183024:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.183028:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.183030:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.183032:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.183049:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+807:3626:x1870152993255808:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3564us (3799us total) trans 4294976375 rc 0/0 00000100:00100000:2.0:1783520936.183057:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44875 00000100:00100000:2.0:1783520936.183082:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002470656 00000100:00100000:2.0:1783520936.183086:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002470656 00000004:00080000:0.0:1783520936.183087:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e9dc00/2, rc 0, transno 4294976375 00000100:00100000:0.0:1783520936.183100:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993255808:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.183106:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003f1b3eb0/0xd5401cbabc021cc7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x83d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f251c expref: 202 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.183115:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.183143:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002470272 00000100:00100000:2.0:1783520936.183147:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002470272 00000100:00100000:1.0:1783520936.183158:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.183158:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993257600:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.183179:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000cba9e215/0xd5401cbabc021baf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x83a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24f2 expref: 203 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.183200:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.183201:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002470528 00000100:00100000:0.0:1783520936.183204:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.183207:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002470528 00010000:00010000:2.0:1783520936.183225:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000074ec2bc/0xd5401cbabc021c1f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x83b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2500 expref: 204 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.183230:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002470784 00000100:00100000:1.0:1783520936.183237:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002470784 00000100:00100000:2.0:1783520936.183251:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36465 00010000:00010000:1.0:1783520936.183259:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007da35a1b/0xd5401cbabc021d7d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x83f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2538 expref: 205 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.183264:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a5880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+205:2387:x1870153002470656:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.183278:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.183284:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003f1b3eb0/0xd5401cbabc021cc7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x83d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f251c expref: 205 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.183290:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36466 00000100:00100000:1.0:1783520936.183302:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a4700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+205:2387:x1870153002470272:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.183313:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003f1b3eb0/0xd5401cbabc021cc7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x83d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f251c expref: 205 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.183314:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.183324:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000cba9e215/0xd5401cbabc021baf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x83a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24f2 expref: 205 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.183334:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003f1b3eb0/0xd5401cbabc021cc7 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x83d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f251c expref: 205 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.183350:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x83d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.183354:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x83d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.183387:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0267ba lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x83d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.183390:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000cba9e215/0xd5401cbabc021baf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x83a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24f2 expref: 204 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.183399:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0267ba lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x83d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.183425:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.183425:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cba9e215/0xd5401cbabc021baf lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x83a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f24f2 expref: 204 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.183440:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36467 00010000:00010000:1.0:1783520936.183440:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x83a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.183447:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x83a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.183452:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0267a5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x83a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.183453:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a5f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+203:2387:x1870153002470528:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.183460:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a5880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+203:2387:x1870153002470656:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 198us (430us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.183468:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.183476:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0267a5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x83a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.183478:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000074ec2bc/0xd5401cbabc021c1f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x83b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2500 expref: 203 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.183480:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36465 00010000:00010000:3.0:1783520936.183513:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000074ec2bc/0xd5401cbabc021c1f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x83b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2500 expref: 202 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.183521:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36468 00010000:00010000:1.0:1783520936.183523:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.183534:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9e680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+202:2387:x1870153002470784:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.183543:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.183545:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000074ec2bc/0xd5401cbabc021c1f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x83b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2500 expref: 202 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.183550:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007da35a1b/0xd5401cbabc021d7d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x83f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2538 expref: 202 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.183554:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a4700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+202:2387:x1870153002470272:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 252us (441us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.183563:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.183570:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x83b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.183578:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x83b:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.183583:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36466 00010000:00010000:3.0:1783520936.183589:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0267ac lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x83b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.183591:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007da35a1b/0xd5401cbabc021d7d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x83f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2538 expref: 201 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.183602:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.183604:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002470912 00000100:00100000:1.0:1783520936.183609:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002470912 00010000:00010000:2.0:1783520936.183616:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007da35a1b/0xd5401cbabc021d7d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x83f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2538 expref: 201 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.183624:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0267ba lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x83d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.183627:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b4c4261c/0xd5401cbabc0228f2 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x83e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fe2 expref: 200 pid: 12734 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.183628:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x83f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.183637:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x83f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.183648:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0267ba lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x83d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.183649:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0267c8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x83f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.183660:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002471040 00000100:00100000:1.0:1783520936.183666:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002471040 00010000:00010000:2.0:1783520936.183674:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0267c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x83f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.183675:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0267ba lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x83d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.183685:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e2608d10/0xd5401cbabc021ddf lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x840:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2546 expref: 201 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.183695:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0267ac lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x83b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.183698:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0267ba lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x83d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.183711:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.183717:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36469 00010000:00010000:0.0:1783520936.183720:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.183722:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.183727:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94361b4a5850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+201:2389:x1870153002470912:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.183731:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.183735:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.183737:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9e680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+201:2387:x1870153002470784:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 207us (540us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.183737:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0267ba lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x83d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.183742:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b4c4261c/0xd5401cbabc0228f2 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x83e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fe2 expref: 201 pid: 12734 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.183751:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a5f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+201:2387:x1870153002470528:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 298us (591us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.183757:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36468 00010000:00010000:0.0:1783520936.183762:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.183767:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b4c4261c/0xd5401cbabc0228f2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x83e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fe2 expref: 201 pid: 12734 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.183768:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0267ba lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x83d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.183781:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36470 00010000:00010000:1.0:1783520936.183787:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b4c4261c/0xd5401cbabc0228f2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x83e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fe2 expref: 200 pid: 12734 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.183795:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502251500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+200:2387:x1870153002471040:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.183798:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x83e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.183802:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x83e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.183802:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0267ba lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x83d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.183804:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.183810:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0267c1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x83e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.183814:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e2608d10/0xd5401cbabc021ddf lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x840:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2546 expref: 199 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.183836:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0267c1 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x83e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.183843:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0267ba lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x83d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.183856:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e2608d10/0xd5401cbabc021ddf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x840:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2546 expref: 199 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.183872:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.183872:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0267ba lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x83d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.183879:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x840:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.183884:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x840:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.183889:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0267cf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x840:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.183891:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.183895:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.183897:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36467 00000100:00100000:1.0:1783520936.183897:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94361b4a5850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+199:2389:x1870153002470912:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 168us (335us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.183898:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.183910:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0267cf lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x840:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.183911:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36469 00000100:00100000:0.0:1783520936.183923:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b07100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993257472:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.183931:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.183944:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc0267d6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x841:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.183961:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502251500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+197:2387:x1870153002471040:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 169us (361us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.183968:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36470 00000100:00100000:3.0:1783520936.184068:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.184091:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+807:3628:x1870152993255936:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4543us (4624us total) trans 4294976376 rc 0/0 00000100:00100000:3.0:1783520936.184099:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002471168 00000100:00100000:0.0:1783520936.184099:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44876 00000100:00100000:3.0:1783520936.184103:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002471168 00010000:00010000:3.0:1783520936.184120:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004e452570/0xd5401cbabc021e1e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x841:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2554 expref: 197 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.184133:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e9d180/2, rc 0, transno 4294976376 00000100:00100000:0.0:1783520936.184141:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993255936:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.184144:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36471 00010000:00010000:1.0:1783520936.184150:0:35282:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e2608d10/0xd5401cbabc021ddf lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x840:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2546 expref: 197 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.184156:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503370e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+197:2390:x1870153002471168:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.184173:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a6a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993257344:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.184182:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502251f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993257728:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.184187:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0267cf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x840:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.184198:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0267cf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x840:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.184206:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0267cf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x840:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.184215:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.184219:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0267cf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x840:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.184221:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.184230:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004e452570/0xd5401cbabc021e1e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x841:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2554 expref: 196 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.184250:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0267cf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x840:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.184255:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004e452570/0xd5401cbabc021e1e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x841:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2554 expref: 196 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.184262:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0267c8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x83f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.184271:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0267c8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x83f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.184276:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004e452570/0xd5401cbabc021e1e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x841:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2554 expref: 196 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.184278:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993257600 00010000:00010000:1.0:1783520936.184280:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.184283:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0267c8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x83f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.184285:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993257600 00010000:00010000:3.0:1783520936.184294:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x841:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.184294:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0267c8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x83f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.184298:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x841:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.184303:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc0267d6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x841:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.184307:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.184310:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc0267d6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x841:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.184312:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0267cf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x840:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.184316:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993257728 00000100:00100000:2.0:1783520936.184321:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993257728 00010000:00010000:0.0:1783520936.184323:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbbe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.184329:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0267cf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x840:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.184338:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0267cf lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x840:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.184349:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44881 00010000:00010000:0.0:1783520936.184350:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0267cf lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x840:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.184379:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0267cf lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x840:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.184398:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435079f2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+809:3628:x1870152993257600:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.184399:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.184402:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.184407:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b05c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+809:3626:x1870152993257088:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2662us (2719us total) trans 4294976377 rc 0/0 00010000:00010000:0.0:1783520936.184407:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.184411:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.184419:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44879 00002000:00080000:2.0:1783520936.184420:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2114 count 1 00000004:00080000:0.0:1783520936.184429:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435079f1180/2, rc 0, transno 4294976377 00000100:00100000:0.0:1783520936.184440:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993257088:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.184442:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503370e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+195:2390:x1870153002471168:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 287us (372us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.184442:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0267dd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x842:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.184442:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44882 00000100:00100000:3.0:1783520936.184458:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36471 00000100:00100000:1.0:1783520936.184459:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502250e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+808:3625:x1870152993257728:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.184465:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003005fd1a/0xd5401cbabc021e48 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x842:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2562 expref: 195 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.184476:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2115 count 1 00010000:00010000:2.0:1783520936.184477:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x842:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.184481:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0267ac lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x83b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.184481:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502250380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993257856:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.184485:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x842:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.184505:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0267e4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x843:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.184512:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0267ac lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x83b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.184522:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e4bd7ac2/0xd5401cbabc021e56 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x843:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2570 expref: 194 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.184523:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.184524:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0267ac lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x83b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.184542:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x843:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.184545:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0267ac lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x83b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.184548:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x843:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.184553:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc0267d6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x841:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.184557:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0267e4 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x843:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.184565:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.184567:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc0267d6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x841:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.184574:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.184579:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0267ac lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x83b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.184581:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc0267d6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x841:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.184587:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.184590:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc0267d6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x841:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.184591:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0267ac lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x83b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.184594:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e4bd7ac2/0xd5401cbabc021e56 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x843:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2570 expref: 194 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.184605:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0267ac lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x83b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.184605:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.184617:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e4bd7ac2/0xd5401cbabc021e56 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x843:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2570 expref: 194 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.184619:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0267dd lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x842:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.184624:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0267ac lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x83b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.184647:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0267ac lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x83b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.184648:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9e680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993257984:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.184662:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003005fd1a/0xd5401cbabc021e48 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x842:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2562 expref: 194 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.184670:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.184675:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc0267d6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x841:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.184676:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003005fd1a/0xd5401cbabc021e48 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x842:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2562 expref: 194 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.184690:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b07b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+809:3628:x1870152993257216:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2816us (2872us total) trans 4294976378 rc 0/0 00010000:00010000:3.0:1783520936.184694:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb800 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520936.184696:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470a80 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.184700:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc0267d6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x841:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.184703:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44880 00000100:00100000:2.0:1783520936.184706:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515476a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993258112:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.184713:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc0267d6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x841:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.184721:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0267a5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x83a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.184726:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc0267d6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x841:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.184732:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513b07480/2, rc 0, transno 4294976378 00010000:00010000:1.0:1783520936.184736:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0267a5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x83a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.184738:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc0267d6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x841:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.184740:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a87a80 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.184742:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b07480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993257216:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.184758:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0267a5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x83a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.184768:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.184769:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0267a5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x83a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.184773:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.184777:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.184784:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350794f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993258240:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.184794:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.184794:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.184797:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.184801:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0267a5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x83a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.184808:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.184812:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0267a5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x83a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.184820:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0267a5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x83a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.184821:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.184831:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0267a5 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x83a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.184836:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993257856 00010000:00010000:3.0:1783520936.184841:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0267a5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x83a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.184841:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993257856 00000100:00100000:0.0:1783520936.184859:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993258240 00000100:00100000:0.0:1783520936.184863:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993258240 00000100:00100000:2.0:1783520936.184868:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44883 00010000:00010000:3.0:1783520936.184874:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.184877:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.184879:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350794ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+811:3626:x1870152993257856:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.184881:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.184887:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44884 00000100:00100000:1.0:1783520936.184894:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+811:3627:x1870152993255424:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6012us (6148us total) trans 4294976379 rc 0/0 00002000:00080000:2.0:1783520936.184895:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2116 count 1 00000100:00100000:1.0:1783520936.184905:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44873 00000004:00080000:0.0:1783520936.184908:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e9f480/2, rc 0, transno 4294976379 00010000:00010000:2.0:1783520936.184912:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0267eb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x844:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.184920:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993255424:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.184930:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000de978b1a/0xd5401cbabc021e64 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x844:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f257e expref: 194 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.184945:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x844:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.184948:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x844:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.184949:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.184954:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0267eb lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x844:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.184976:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:3.0:1783520936.184978:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+811:3625:x1870152993255552:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5786us (5896us total) trans 4294976380 rc 0/0 00010000:00010000:0.0:1783520936.184981:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0267c8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x83f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.184987:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44874 00010000:00010000:2.0:1783520936.184988:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000de978b1a/0xd5401cbabc021e64 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x844:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f257e expref: 194 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.184991:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.184998:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0267c8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x83f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.185006:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000de978b1a/0xd5401cbabc021e64 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x844:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f257e expref: 194 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.185010:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0267c8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x83f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.185012:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0267c1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x83e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.185015:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+811:3625:x1870152993256832:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4695us (4767us total) trans 4294976381 rc 0/0 00010000:00010000:3.0:1783520936.185032:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0267c1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x83e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.185033:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515477800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993258496:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.185034:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44878 00010000:00010000:0.0:1783520936.185037:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0267c8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x83f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.185049:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0267c1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x83e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.185049:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0267c8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x83f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.185061:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0267c1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x83e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.185066:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.185068:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943607eb8300 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.185069:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.185075:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.185076:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520936.185080:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:0.0:1783520936.185081:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0267c1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x83e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.185084:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.185093:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515345500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993258368:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.185101:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.185106:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0267c1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x83e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.185114:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0267c1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x83e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.185124:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0267c1 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x83e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.185134:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.185137:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc0267c1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x83e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.185154:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.185157:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.185159:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.185167:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350794df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+811:3627:x1870152993258240:12345-0@lo:6:osp-syn-1-0.0 00000004:00080000:0.0:1783520936.185181:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435121a6300/2, rc 0, transno 4294976381 00002000:00080000:1.0:1783520936.185182:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2117 count 1 00000100:00100000:0.0:1783520936.185194:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993256832:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.185198:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0267f2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x845:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.185208:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000059f9ff6b/0xd5401cbabc021e80 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x845:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f258c expref: 194 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.185213:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x845:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.185217:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x845:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.185220:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0267f2 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x845:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.185232:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435079f1500/2, rc 0, transno 4294976380 00010000:00010000:1.0:1783520936.185237:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000059f9ff6b/0xd5401cbabc021e80 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x845:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f258c expref: 194 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.185241:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+811:3628:x1870152993256576:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5210us (5277us total) trans 4294976382 rc 0/0 00000100:00100000:0.0:1783520936.185241:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993255552:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.185253:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000059f9ff6b/0xd5401cbabc021e80 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x845:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f258c expref: 194 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.185254:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44877 00000100:00100000:1.0:1783520936.185265:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b04e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993258752:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.185281:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993258880:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.185291:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470240 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.185312:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.185313:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993258368 00000100:00100000:3.0:1783520936.185317:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993258368 00000100:00100000:3.0:1783520936.185339:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993258880 00000100:00100000:3.0:1783520936.185342:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993258880 00000100:00100000:0.0:1783520936.185345:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e1c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993258624:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.185352:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44885 00000100:00100000:1.0:1783520936.185383:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b07100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+812:3627:x1870152993258368:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.185393:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00080000:1.0:1783520936.185397:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2118 count 1 00010000:00010000:1.0:1783520936.185411:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0267f9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x846:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.185420:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993258624 00010000:00010000:1.0:1783520936.185423:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c937e328/0xd5401cbabc022b4c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x846:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f323c expref: 194 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.185425:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993258624 00000100:00100000:2.0:1783520936.185430:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44886 00010000:00010000:1.0:1783520936.185434:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x846:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.185437:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x846:0x0].0x0 since=0s v=(1 0) 00000100:00100000:2.0:1783520936.185440:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+813:3626:x1870152993258880:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.185442:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0267f9 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x846:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.185452:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44887 00002000:00080000:2.0:1783520936.185453:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2120 count 1 00000004:00080000:0.0:1783520936.185454:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e9ea00/2, rc 0, transno 4294976382 00000100:00100000:3.0:1783520936.185460:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+813:3625:x1870152993258624:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.185465:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c937e328/0xd5401cbabc022b4c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x846:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f323c expref: 194 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.185465:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993256576:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.185469:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026800 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x848:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.185476:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2119 count 1 00010000:00010000:2.0:1783520936.185483:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008d75ff72/0xd5401cbabc021e95 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x848:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f25b6 expref: 194 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.185483:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c937e328/0xd5401cbabc022b4c lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x846:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f323c expref: 194 pid: 8410 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.185503:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026807 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x847:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.185505:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x848:0x0].0x0 since=38s v=(0 0) 00000100:00100000:1.0:1783520936.185509:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a6a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993259008:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.185511:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x848:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.185519:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993259136:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.185521:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026800 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x848:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.185526:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000000292b55e/0xd5401cbabc021e8e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x847:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f25a8 expref: 194 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.185547:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008d75ff72/0xd5401cbabc021e95 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x848:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25b6 expref: 194 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.185552:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470900 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.185557:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.185567:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008d75ff72/0xd5401cbabc021e95 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x848:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25b6 expref: 194 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.185573:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993259136 00000100:00100000:1.0:1783520936.185576:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993259136 00000100:00100000:2.0:1783520936.185579:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515476680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993259264:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.185599:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x847:0x0].0x0 since=38s v=(0 0) 00000100:00100000:2.0:1783520936.185604:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b19c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.185604:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44888 00010000:00010000:3.0:1783520936.185606:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x847:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.185611:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026807 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x847:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.185615:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+814:3627:x1870152993259136:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.185633:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2121 count 1 00010000:00010000:1.0:1783520936.185647:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02680e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x849:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.185655:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000000292b55e/0xd5401cbabc021e8e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x847:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25a8 expref: 194 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.185681:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000000292b55e/0xd5401cbabc021e8e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x847:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25a8 expref: 194 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.185681:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000db888310/0xd5401cbabc021ebf lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x849:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f25c4 expref: 194 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.185701:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x849:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.185705:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x849:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.185707:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503370380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993259392:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.185710:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02680e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x849:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.185734:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000db888310/0xd5401cbabc021ebf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x849:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25c4 expref: 194 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.185745:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000db888310/0xd5401cbabc021ebf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x849:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25c4 expref: 194 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.185756:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a5880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993259520:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.185777:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57d80 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.185789:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470840 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.186567:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9e680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993257984:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.186584:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0267e4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x843:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.186641:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515476a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993258112:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.186654:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0267dd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x842:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.187399:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.187455:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002471296 00000100:00100000:3.0:1783520936.187461:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002471296 00010000:00010000:3.0:1783520936.187504:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e4bd7ac2/0xd5401cbabc021e56 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x843:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2570 expref: 195 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.187504:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b04e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993258752:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.187527:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0267f2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x845:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.187549:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36472 00000100:00100000:2.0:1783520936.187585:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.187607:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002471424 00000100:00100000:0.0:1783520936.187613:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002471424 00010000:00010000:0.0:1783520936.187634:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003005fd1a/0xd5401cbabc021e48 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x842:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2562 expref: 196 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.187658:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36473 00000100:00100000:0.0:1783520936.187664:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515476d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+196:2390:x1870153002471424:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.187673:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.187677:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003005fd1a/0xd5401cbabc021e48 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x842:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2562 expref: 196 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.187681:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515476680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993259264:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.187689:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003005fd1a/0xd5401cbabc021e48 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x842:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2562 expref: 196 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.187696:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515474e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+196:2389:x1870153002471296:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.187700:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026800 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x848:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.187707:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.187711:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003005fd1a/0xd5401cbabc021e48 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x842:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2562 expref: 196 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.187716:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e4bd7ac2/0xd5401cbabc021e56 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x843:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2570 expref: 196 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.187732:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x842:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.187734:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e4bd7ac2/0xd5401cbabc021e56 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x843:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2570 expref: 195 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.187739:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x842:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.187745:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0267dd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x842:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.187760:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0267dd lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x842:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.187785:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.187806:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503370380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993259392:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.187823:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e4bd7ac2/0xd5401cbabc021e56 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x843:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2570 expref: 195 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.187835:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x843:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.187838:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x843:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.187842:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0267e4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x843:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.187850:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0267e4 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x843:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.187886:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.187891:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026807 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x847:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.187911:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515476d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+194:2390:x1870153002471424:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 244us (324us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.187916:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0267dd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x842:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.187918:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.187931:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36473 00010000:00010000:3.0:1783520936.187932:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0267dd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x842:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.187949:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002471552 00000100:00100000:0.0:1783520936.187952:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002471552 00010000:00010000:0.0:1783520936.187963:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000059f9ff6b/0xd5401cbabc021e80 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x845:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f258c expref: 194 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.187980:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002471808 00000100:00100000:0.0:1783520936.187983:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002471808 00010000:00010000:0.0:1783520936.187996:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008d75ff72/0xd5401cbabc021e95 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x848:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25b6 expref: 195 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.188002:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.188008:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0267dd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x842:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.188020:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0267dd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x842:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.188020:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36474 00000100:00100000:0.0:1783520936.188031:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515475880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+195:2390:x1870153002471552:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.188040:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.188045:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000059f9ff6b/0xd5401cbabc021e80 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x845:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f258c expref: 195 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.188056:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000059f9ff6b/0xd5401cbabc021e80 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x845:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f258c expref: 195 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.188060:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.188081:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.188091:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.188099:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.188103:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000059f9ff6b/0xd5401cbabc021e80 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x845:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f258c expref: 195 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.188117:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.188117:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x845:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.188123:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x845:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.188128:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0267f2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x845:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.188135:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002472064 00010000:00010000:0.0:1783520936.188137:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0267f2 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x845:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.188141:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002472064 00010000:00010000:0.0:1783520936.188156:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.188160:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000de978b1a/0xd5401cbabc021e64 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x844:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f257e expref: 195 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.188163:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0267dd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x842:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.188175:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.188181:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0267dd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x842:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.188193:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0267dd lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x842:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.188194:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36475 00000100:00100000:2.0:1783520936.188203:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515475500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+195:2390:x1870153002471808:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.188211:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.188219:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008d75ff72/0xd5401cbabc021e95 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x848:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25b6 expref: 195 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.188223:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0267dd lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x842:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.188231:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435079f2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+815:3628:x1870152993257600:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3872us (5029us total) trans 4294976383 rc 0/0 00010000:00010000:2.0:1783520936.188234:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008d75ff72/0xd5401cbabc021e95 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x848:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25b6 expref: 195 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.188250:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44881 00010000:00010000:0.0:1783520936.188260:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc0267dd lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x842:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.188267:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008d75ff72/0xd5401cbabc021e95 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x848:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25b6 expref: 195 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.188280:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002472192 00010000:00010000:2.0:1783520936.188281:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x848:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.188286:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x848:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.188287:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002472192 00010000:00010000:2.0:1783520936.188291:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026800 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x848:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.188298:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.188300:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026800 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x848:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.188302:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.188307:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.188308:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000db888310/0xd5401cbabc021ebf lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x849:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25c4 expref: 195 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.188323:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.188329:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002471680 00000100:00100000:3.0:1783520936.188334:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002471680 00000004:00080000:0.0:1783520936.188339:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435079f0380/2, rc 0, transno 4294976383 00010000:00010000:3.0:1783520936.188350:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c937e328/0xd5401cbabc022b4c lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x846:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f323c expref: 196 pid: 8410 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.188351:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515475500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+196:2390:x1870153002471808:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 148us (436us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.188355:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993257600:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.188385:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36475 00000100:00100000:2.0:1783520936.188403:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002471936 00000100:00100000:2.0:1783520936.188406:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002471936 00010000:00010000:2.0:1783520936.188422:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000000292b55e/0xd5401cbabc021e8e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x847:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25a8 expref: 196 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.188460:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36476 00000100:00100000:2.0:1783520936.188474:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94361b4a4050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+196:2389:x1870153002472064:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.188482:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.188498:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000de978b1a/0xd5401cbabc021e64 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x844:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f257e expref: 196 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.188504:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:2.0:1783520936.188509:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000de978b1a/0xd5401cbabc021e64 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x844:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f257e expref: 196 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.188521:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993259648:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.188524:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x844:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.188528:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x844:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.188533:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0267eb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x844:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.188540:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0267eb lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x844:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.188553:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.188554:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.188567:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36477 00000100:00100000:3.0:1783520936.188578:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515476680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+196:2389:x1870153002472192:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.188584:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94361b4a4050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+196:2389:x1870153002472064:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 115us (524us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.188590:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.188595:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36476 00010000:00010000:3.0:1783520936.188600:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000db888310/0xd5401cbabc021ebf lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x849:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25c4 expref: 196 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.188605:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.188615:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000db888310/0xd5401cbabc021ebf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x849:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25c4 expref: 195 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.188619:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515474e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+195:2389:x1870153002471296:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 1053us (1218us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.188620:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36478 00000100:00100000:0.0:1783520936.188635:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515475880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+195:2390:x1870153002471552:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 602us (746us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.188636:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515476a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+195:2388:x1870153002471680:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.188637:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36472 00010000:00010000:3.0:1783520936.188645:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x849:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.188649:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.188651:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x849:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.188652:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36474 00010000:00010000:2.0:1783520936.188658:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c937e328/0xd5401cbabc022b4c lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x846:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f323c expref: 194 pid: 8410 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.188671:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c937e328/0xd5401cbabc022b4c lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x846:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f323c expref: 193 pid: 8410 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.188678:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36479 00010000:00010000:2.0:1783520936.188686:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x846:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.188691:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x846:0x0].0x0 since=0s v=(1 0) 00000100:00100000:1.0:1783520936.188694:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515474000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+193:2388:x1870153002471936:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.188697:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0267f9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x846:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.188706:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.188707:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993259648 00010000:00010000:2.0:1783520936.188708:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0267f9 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x846:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.188713:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993259648 00010000:00010000:1.0:1783520936.188716:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000000292b55e/0xd5401cbabc021e8e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x847:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25a8 expref: 193 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.188719:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02680e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x849:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.188729:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.188742:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02680e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x849:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.188746:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44889 00000100:00100000:2.0:1783520936.188754:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515476a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+193:2388:x1870153002471680:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 120us (652us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.188756:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000000292b55e/0xd5401cbabc021e8e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x847:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25a8 expref: 193 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.188762:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+815:3627:x1870152993259648:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.188773:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36478 00010000:00010000:3.0:1783520936.188774:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.188784:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000292b55e/0xd5401cbabc021e8e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x847:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25a8 expref: 193 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.188788:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2122 count 1 00000100:00100000:3.0:1783520936.188801:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515476680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+191:2389:x1870153002472192:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 224us (720us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.188802:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x847:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.188808:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026815 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x84a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.188811:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x847:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.188819:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36477 00010000:00010000:1.0:1783520936.188830:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026807 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x847:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.188832:0:178496:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000de978b1a/0xd5401cbabc021e64 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x844:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f257e expref: 190 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.188833:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000247e5551/0xd5401cbabc021eb1 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x84a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f25d2 expref: 190 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.188845:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026800 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x848:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.188857:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026807 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x847:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.188865:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515477800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993258496:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.188882:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x84a:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.188885:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0267eb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x844:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.188886:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026800 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x848:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.188887:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x84a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.188890:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.188898:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026815 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x84a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.188911:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515474000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+189:2388:x1870153002471936:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 220us (792us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.188912:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0267eb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x844:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.188914:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026800 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x848:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.188927:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36479 00010000:00010000:0.0:1783520936.188934:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000247e5551/0xd5401cbabc021eb1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x84a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25d2 expref: 189 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.188941:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0267eb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x844:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.188942:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026800 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x848:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.188951:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000247e5551/0xd5401cbabc021eb1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x84a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25d2 expref: 188 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.188967:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0267eb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x844:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.188975:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.188976:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993259776:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.188979:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0267eb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x844:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.188989:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.188999:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.189002:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026800 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x848:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.189014:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.189018:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026800 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x848:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.189027:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026800 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x848:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.189031:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943507895840 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.189045:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026800 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x848:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.189053:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026800 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x848:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.189065:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0267f2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x845:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.189074:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.189077:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0267f2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x845:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.189079:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0267eb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x844:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.189095:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0267f2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x845:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.189103:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350794ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+816:3626:x1870152993257856:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4226us (4578us total) trans 4294976384 rc 0/0 00010000:00010000:1.0:1783520936.189109:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0267f2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x845:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.189116:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44883 00010000:00010000:1.0:1783520936.189133:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.189146:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0267e4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x843:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.189147:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502250380/2, rc 0, transno 4294976384 00000100:00100000:0.0:1783520936.189162:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502250380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993257856:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.189169:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0267e4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x843:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.189172:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.189179:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0267eb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x844:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.189181:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.189183:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0267e4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x843:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.189191:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0267f2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x845:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.189199:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0267eb lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x844:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.189210:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350794df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+816:3627:x1870152993258240:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4313us (4386us total) trans 4294976385 rc 0/0 00010000:00010000:2.0:1783520936.189213:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0267e4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x843:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.189228:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44884 00010000:00010000:0.0:1783520936.189229:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.189236:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0267f2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x845:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.189239:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.189252:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0267f2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x845:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.189260:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0267eb lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x844:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.189268:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0267f2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x845:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.189269:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0267eb lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x844:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.189278:0:35276:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c937e328/0xd5401cbabc022b4c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x846:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f323c expref: 188 pid: 8410 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.189294:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc0267f2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x845:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.189302:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a6a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993259008:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.189311:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.189316:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0267e4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x843:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.189321:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.189322:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0267f9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x846:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.189326:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502250e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+816:3625:x1870152993257728:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4870us (5111us total) trans 4294976386 rc 0/0 00010000:00010000:0.0:1783520936.189326:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.189328:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.189331:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.189337:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0267e4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x843:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.189341:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.189344:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0267f9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x846:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.189347:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44882 00010000:00010000:1.0:1783520936.189389:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0267f9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x846:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.189399:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0267e4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x843:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.189400:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0267f9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x846:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.189402:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1783520936.189416:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993259904:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.189424:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0267f9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x846:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.189428:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0267e4 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x843:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.189434:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502251f80/2, rc 0, transno 4294976386 00010000:00010000:3.0:1783520936.189445:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc0267e4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x843:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.189452:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502251f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993257728:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.189452:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.189453:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.189477:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.189484:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0267f9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x846:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.189504:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:0.0:1783520936.189506:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.189512:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0267f9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x846:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.189517:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515477800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993260032:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.189527:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0267f9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x846:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.189545:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0267f9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x846:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.189547:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.189551:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.189554:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b07100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+816:3627:x1870152993258368:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4193us (4421us total) trans 4294976387 rc 0/0 00010000:00010000:3.0:1783520936.189556:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.189562:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.189566:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.189572:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44885 00010000:00010000:0.0:1783520936.189575:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc0267f9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x846:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.189597:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993259904 00000100:00100000:1.0:1783520936.189601:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993259904 00010000:00010000:0.0:1783520936.189601:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.189606:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.189610:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.189620:0:8410:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000db888310/0xd5401cbabc021ebf lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x849:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25c4 expref: 187 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.189633:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993260032 00000100:00100000:2.0:1783520936.189637:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a5880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993259520:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.189638:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993260032 00000004:00080000:0.0:1783520936.189651:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515345500/2, rc 0, transno 4294976387 00010000:00010000:2.0:1783520936.189656:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02680e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x849:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.189668:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02680e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x849:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.189668:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44890 00000100:00100000:0.0:1783520936.189668:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515345500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993258368:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.189679:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+818:3626:x1870152993259904:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.189688:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02680e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x849:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.189694:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2123 count 1 00010000:00010000:2.0:1783520936.189706:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02680e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x849:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.189708:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02681c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x84b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.189728:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02680e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x849:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.189730:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c1acd2c5/0xd5401cbabc021ecd lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x84b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f25e0 expref: 186 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.189737:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e1c7800/2, rc 0, transno 4294976388 00000100:00100000:3.0:1783520936.189741:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+818:3626:x1870152993258880:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4299us (4425us total) trans 4294976388 rc 0/0 00010000:00010000:1.0:1783520936.189751:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x84b:0x0].0x0 since=38s v=(0 0) 00000100:00100000:0.0:1783520936.189754:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993258880:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.189755:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x84b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.189757:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520936.189759:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44886 00010000:00010000:1.0:1783520936.189761:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02681c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x84b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.189783:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c1acd2c5/0xd5401cbabc021ecd lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x84b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25e0 expref: 186 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.189787:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44891 00010000:00010000:0.0:1783520936.189796:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.189800:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c1acd2c5/0xd5401cbabc021ecd lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x84b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25e0 expref: 186 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.189802:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515474700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+818:3625:x1870152993260032:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.189805:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02680e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x849:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.189821:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2124 count 1 00000100:00100000:1.0:1783520936.189829:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a6a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993260288:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.189829:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.189836:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02680e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x849:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.189838:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026823 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x84c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.189841:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+818:3627:x1870152993259136:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4227us (4284us total) trans 4294976389 rc 0/0 00000100:00100000:2.0:1783520936.189860:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44888 00000100:00100000:1.0:1783520936.189863:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470780 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.189864:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02680e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x849:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.189870:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000faf075ec/0xd5401cbabc021ed4 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x84c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f25ee expref: 186 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.189885:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02680e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x849:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.189889:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x84c:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.189896:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x84c:0x0].0x0 since=0s v=(0 0) 00000004:00080000:1.0:1783520936.189898:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350794f480/2, rc 0, transno 4294976385 00010000:00010000:0.0:1783520936.189899:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc02680e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x849:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.189903:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026823 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x84c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.189907:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350794f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993258240:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.189922:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993260160:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.189939:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.189942:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000faf075ec/0xd5401cbabc021ed4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x84c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25ee expref: 186 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.189943:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.189949:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.189955:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.189959:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000faf075ec/0xd5401cbabc021ed4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x84c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25ee expref: 186 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.189975:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503370380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993260544:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.189985:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993260160 00000004:00080000:1.0:1783520936.189990:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b3b80/2, rc 0, transno 4294976389 00000100:00100000:2.0:1783520936.189991:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993260160 00000100:00100000:0.0:1783520936.189994:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993260672:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.190005:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993259136:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.190015:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1783520936.190024:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502251500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993260416:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.190024:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.190029:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44892 00000100:00100000:2.0:1783520936.190041:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+818:3627:x1870152993260160:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.190055:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2125 count 1 00000100:00100000:3.0:1783520936.190059:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993260672 00000100:00100000:3.0:1783520936.190065:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993260672 00000100:00100000:1.0:1783520936.190066:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.190070:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02682a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x84d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.190085:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001b96c0d2/0xd5401cbabc021ee2 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x84d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f25fc expref: 186 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.190091:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:2.0:1783520936.190093:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x84d:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.190098:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x84d:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.190101:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993260416 00000100:00100000:0.0:1783520936.190101:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993260800:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.190104:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02682a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x84d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.190105:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44893 00000100:00100000:3.0:1783520936.190108:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993260416 00000100:00100000:1.0:1783520936.190117:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+820:3625:x1870152993260672:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.190134:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2127 count 1 00000100:00100000:3.0:1783520936.190136:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44894 00000100:00100000:0.0:1783520936.190139:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.190147:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001b96c0d2/0xd5401cbabc021ee2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x84d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25fc expref: 186 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.190147:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026831 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x84f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.190153:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+820:3627:x1870152993260416:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.190164:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993260800 00000100:00100000:0.0:1783520936.190168:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993260800 00010000:00010000:1.0:1783520936.190172:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000019de2428/0xd5401cbabc021efe lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x84f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2618 expref: 186 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.190174:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001b96c0d2/0xd5401cbabc021ee2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x84d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25fc expref: 186 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.190175:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2126 count 1 00010000:00010000:1.0:1783520936.190183:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x84f:0x0].0x0 since=38s v=(0 0) 00000100:00100000:0.0:1783520936.190188:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44895 00010000:00010000:3.0:1783520936.190193:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026838 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x84e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.190193:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x84f:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.190200:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+821:3625:x1870152993260800:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.190201:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515474380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993260928:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.190203:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026831 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x84f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.190217:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2128 count 1 00010000:00010000:3.0:1783520936.190224:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000000b160cc6/0xd5401cbabc022a3b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x84e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f312b expref: 186 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.190231:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02683f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x850:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.190241:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000019de2428/0xd5401cbabc021efe lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x84f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2618 expref: 186 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.190245:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1180 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.190258:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x84e:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.190261:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000084b40457/0xd5401cbabc021f05 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x850:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2626 expref: 186 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.190262:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000019de2428/0xd5401cbabc021efe lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x84f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2618 expref: 186 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.190267:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x84e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.190277:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026838 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x84e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.190288:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x850:0x0].0x0 since=38s v=(0 0) 00000100:00100000:1.0:1783520936.190292:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a5880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993261056:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.190296:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x850:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.190302:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02683f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x850:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.190325:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470e40 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.190327:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000084b40457/0xd5401cbabc021f05 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x850:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2626 expref: 186 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.190336:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000084b40457/0xd5401cbabc021f05 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x850:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2626 expref: 186 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.190344:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507acf100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993261184:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.190390:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff9436298a6480 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.190419:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026807 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x847:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.190430:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026807 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x847:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.190438:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026807 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x847:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.190446:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026807 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x847:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.190464:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.190475:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.190480:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026807 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x847:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.190486:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.190504:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026807 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x847:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.190508:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000000b160cc6/0xd5401cbabc022a3b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x84e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f312b expref: 186 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.190513:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.190526:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026807 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x847:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.190533:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002472320 00000100:00100000:0.0:1783520936.190536:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002472320 00010000:00010000:1.0:1783520936.190542:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026807 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x847:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.190551:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026807 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x847:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.190552:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000247e5551/0xd5401cbabc021eb1 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x84a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25d2 expref: 187 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.190560:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000000b160cc6/0xd5401cbabc022a3b lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x84e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f312b expref: 187 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.190568:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+822:3625:x1870152993258624:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5105us (5171us total) trans 4294976390 rc 0/0 00000100:00100000:2.0:1783520936.190585:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44887 00000100:00100000:0.0:1783520936.190585:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36480 00000100:00100000:0.0:1783520936.190596:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507acd500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+187:2390:x1870153002472320:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.190597:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.190601:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.190602:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.190604:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.190608:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000247e5551/0xd5401cbabc021eb1 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x84a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25d2 expref: 187 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.190615:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000247e5551/0xd5401cbabc021eb1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x84a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25d2 expref: 187 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.190625:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x84a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.190627:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x84a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.190630:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026815 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.190637:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e1c7100/2, rc 0, transno 4294976390 00010000:00010000:0.0:1783520936.190639:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026815 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x84a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.190650:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e1c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993258624:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.190674:0:35274:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000247e5551/0xd5401cbabc021eb1 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x84a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25d2 expref: 187 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.190679:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507acc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993261440:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.190685:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b1f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993259776:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.190702:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026815 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x84a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.190712:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.190714:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503371c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993261312:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.190736:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.190756:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a87a80 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.190758:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507acd500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+186:2390:x1870153002472320:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 164us (248us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.190762:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993261440 00000100:00100000:0.0:1783520936.190765:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993261440 00000100:00100000:1.0:1783520936.190766:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36480 00000100:00100000:0.0:1783520936.190799:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44896 00010000:00010000:1.0:1783520936.190800:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026815 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.190807:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635784850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+822:3627:x1870152993261440:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.190815:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026815 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.190821:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2129 count 1 00010000:00010000:1.0:1783520936.190824:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026815 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.190835:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026846 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x851:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.190840:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026815 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x84a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.190856:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000054c81770/0xd5401cbabc021f28 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x851:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2634 expref: 185 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.190864:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x851:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.190867:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.190867:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x851:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.190874:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026846 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x851:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.190876:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.190885:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026815 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x84a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.190897:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.190898:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000054c81770/0xd5401cbabc021f28 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x851:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2634 expref: 185 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.190903:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026815 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x84a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.190909:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000054c81770/0xd5401cbabc021f28 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x851:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2634 expref: 185 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.190918:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507acea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993261568:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.190953:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026815 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x84a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.190958:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+823:3627:x1870152993259648:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2200us (2404us total) trans 4294976391 rc 0/0 00000100:00100000:1.0:1783520936.190965:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44889 00000004:00080000:0.0:1783520936.190967:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b3480/2, rc 0, transno 4294976391 00000100:00100000:0.0:1783520936.190978:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993259648:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.190996:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a6a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993260288:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.191001:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026815 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x84a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.191010:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507acf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993261696:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.191011:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02681c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x84b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.191013:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc026815 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x84a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.191037:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.191044:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.191047:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80840 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.191049:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.191054:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.191060:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993261696 00000100:00100000:1.0:1783520936.191063:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993261696 00000100:00100000:3.0:1783520936.191087:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57d80 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.191091:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44897 00000100:00100000:1.0:1783520936.191100:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ae5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+823:3628:x1870152993261696:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.191112:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2130 count 1 00010000:00010000:1.0:1783520936.191124:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02684d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x852:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.191136:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005d429a2f/0xd5401cbabc021f2f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x852:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2642 expref: 185 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.191143:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x852:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.191146:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x852:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.191150:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02684d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x852:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.191168:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005d429a2f/0xd5401cbabc021f2f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x852:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2642 expref: 185 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.191178:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005d429a2f/0xd5401cbabc021f2f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x852:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2642 expref: 185 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.191188:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a6a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993261824:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.191219:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470780 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.191613:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.191664:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002472448 00000100:00100000:3.0:1783520936.191669:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002472448 00010000:00010000:3.0:1783520936.191687:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c1acd2c5/0xd5401cbabc021ecd lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x84b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25e0 expref: 186 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.191709:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36481 00000100:00100000:3.0:1783520936.191715:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515477b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+186:2387:x1870153002472448:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.191721:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.191724:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c1acd2c5/0xd5401cbabc021ecd lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x84b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25e0 expref: 186 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.191731:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c1acd2c5/0xd5401cbabc021ecd lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x84b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25e0 expref: 186 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.191741:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c1acd2c5/0xd5401cbabc021ecd lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x84b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25e0 expref: 186 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.191749:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x84b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.191751:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x84b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.191754:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02681c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.191760:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02681c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x84b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.191777:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.191809:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515477b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+185:2387:x1870153002472448:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 92us (194us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.191809:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02681c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.191825:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36481 00010000:00010000:1.0:1783520936.191831:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02681c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.191843:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02681c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.191851:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02681c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x84b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.191874:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.191895:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.191901:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02681c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x84b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.191910:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.191914:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02681c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x84b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.191923:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02681c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x84b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.191938:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02681c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x84b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.191946:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc02681c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x84b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.191956:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+824:3626:x1870152993259904:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2277us (2506us total) trans 4294976392 rc 0/0 00010000:00010000:2.0:1783520936.191967:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.191968:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44890 00010000:00010000:2.0:1783520936.191972:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520936.191975:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b1c00/2, rc 0, transno 4294976392 00010000:00010000:2.0:1783520936.191975:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.191996:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993259904:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.192654:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ae6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993261952:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.192686:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.192760:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993261952 00000100:00100000:2.0:1783520936.192765:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993261952 00000100:00100000:2.0:1783520936.192826:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44898 00000100:00100000:2.0:1783520936.192834:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+824:3625:x1870152993261952:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.192849:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2131 count 1 00010000:00010000:2.0:1783520936.192862:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026854 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x853:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.192876:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007d2d8f28/0xd5401cbabc021f36 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x853:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2650 expref: 184 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.192883:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x853:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.192886:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x853:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.192890:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026854 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x853:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.192907:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007d2d8f28/0xd5401cbabc021f36 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x853:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2650 expref: 184 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.192914:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007d2d8f28/0xd5401cbabc021f36 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x853:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2650 expref: 184 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.192922:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515476d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993262080:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.192959:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1480 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.193092:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515474380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993260928:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.193111:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02682a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x84d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.193144:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a5880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993261056:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.193157:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026831 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x84f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.193244:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.193272:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.193297:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.193311:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002472576 00000100:00100000:1.0:1783520936.193315:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002472576 00010000:00010000:1.0:1783520936.193330:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000019de2428/0xd5401cbabc021efe lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x84f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2618 expref: 185 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.193348:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36482 00000100:00100000:1.0:1783520936.193355:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503372300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+185:2387:x1870153002472576:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.193389:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507acf100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993261184:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.193397:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.193404:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000019de2428/0xd5401cbabc021efe lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x84f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2618 expref: 185 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.193415:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000019de2428/0xd5401cbabc021efe lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x84f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2618 expref: 185 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.193419:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02683f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x850:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.193435:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000019de2428/0xd5401cbabc021efe lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x84f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2618 expref: 185 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.193443:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a6a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993261824:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.193458:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02684d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x852:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.193458:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x84f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.193461:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x84f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.193467:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026831 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.193474:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.193483:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026831 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x84f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.193518:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.193523:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.193535:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002472704 00000100:00100000:0.0:1783520936.193541:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002472704 00000100:00100000:1.0:1783520936.193547:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503372300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+185:2387:x1870153002472576:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 191us (359us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.193555:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36482 00000100:00100000:2.0:1783520936.193556:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002472832 00000100:00100000:2.0:1783520936.193560:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002472832 00010000:00010000:0.0:1783520936.193563:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001b96c0d2/0xd5401cbabc021ee2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x84d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25fc expref: 185 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.193568:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002472960 00000100:00100000:1.0:1783520936.193572:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002472960 00010000:00010000:2.0:1783520936.193579:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000084b40457/0xd5401cbabc021f05 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x850:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2626 expref: 186 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.193584:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000faf075ec/0xd5401cbabc021ed4 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x84c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25ee expref: 186 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.193601:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002473088 00000100:00100000:2.0:1783520936.193603:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002473088 00000100:00100000:1.0:1783520936.193609:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36483 00010000:00010000:2.0:1783520936.193619:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000054c81770/0xd5401cbabc021f28 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x851:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2634 expref: 187 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.193621:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94361b4a6050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+187:2389:x1870153002472704:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.193621:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36484 00000100:00100000:3.0:1783520936.193626:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.193633:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.193641:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001b96c0d2/0xd5401cbabc021ee2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x84d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25fc expref: 187 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.193641:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+187:2390:x1870153002472832:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.193642:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002473216 00000100:00100000:2.0:1783520936.193646:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002473216 00010000:00010000:1.0:1783520936.193657:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001b96c0d2/0xd5401cbabc021ee2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x84d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25fc expref: 188 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.193658:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.193659:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005d429a2f/0xd5401cbabc021f2f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x852:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2642 expref: 188 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.193671:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000084b40457/0xd5401cbabc021f05 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x850:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2626 expref: 188 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.193676:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001b96c0d2/0xd5401cbabc021ee2 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x84d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25fc expref: 188 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.193677:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36485 00000100:00100000:2.0:1783520936.193686:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503373480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+188:2388:x1870153002472960:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.193698:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.193699:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x84d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.193701:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000084b40457/0xd5401cbabc021f05 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x850:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2626 expref: 187 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.193703:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x84d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.193704:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000faf075ec/0xd5401cbabc021ed4 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x84c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25ee expref: 187 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.193710:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02682a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.193729:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000faf075ec/0xd5401cbabc021ed4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x84c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25ee expref: 187 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.193729:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02682a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x84d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.193749:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x84c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.193750:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000084b40457/0xd5401cbabc021f05 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x850:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2626 expref: 187 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.193752:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.193755:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x84c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.193763:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026823 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.193773:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026823 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x84c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.193773:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x850:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.193782:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x850:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.193789:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02683f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x850:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.193791:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.193797:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94361b4a6050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+186:2389:x1870153002472704:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 180us (523us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.193808:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503373480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+186:2388:x1870153002472960:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 123us (339us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.193808:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36483 00010000:00010000:0.0:1783520936.193808:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02683f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x850:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.193824:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36485 00000100:00100000:2.0:1783520936.193838:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36486 00010000:00010000:0.0:1783520936.193845:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.193846:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503370e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+184:2389:x1870153002473088:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.193848:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36487 00010000:00010000:2.0:1783520936.193855:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.193855:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503372a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+184:2387:x1870153002473216:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.193860:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000054c81770/0xd5401cbabc021f28 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x851:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2634 expref: 184 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.193862:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.193867:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005d429a2f/0xd5401cbabc021f2f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x852:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2642 expref: 184 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.193871:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000054c81770/0xd5401cbabc021f28 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x851:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2634 expref: 184 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.193877:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+184:2390:x1870153002472832:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 241us (575us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.193879:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005d429a2f/0xd5401cbabc021f2f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x852:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2642 expref: 184 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.193887:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x851:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.193893:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x851:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.193897:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.193900:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36484 00010000:00010000:2.0:1783520936.193901:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026846 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x851:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.193911:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026846 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x851:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.193914:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005d429a2f/0xd5401cbabc021f2f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x852:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2642 expref: 184 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.193926:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002473344 00010000:00010000:2.0:1783520936.193927:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.193927:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x852:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.193932:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x852:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.193933:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002473344 00010000:00010000:1.0:1783520936.193937:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02684d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x852:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.193946:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02684d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x852:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.193952:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000000b160cc6/0xd5401cbabc022a3b lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x84e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f312b expref: 183 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.193964:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.193980:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36488 00000100:00100000:0.0:1783520936.193989:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+183:2389:x1870153002473344:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.193998:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.194004:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000000b160cc6/0xd5401cbabc022a3b lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x84e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f312b expref: 183 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.194017:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000000b160cc6/0xd5401cbabc022a3b lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x84e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f312b expref: 183 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.194025:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503370e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+183:2389:x1870153002473088:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 179us (501us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.194025:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503372a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+183:2387:x1870153002473216:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 168us (403us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.194035:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36487 00010000:00010000:0.0:1783520936.194035:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x84e:0x0].0x0 since=0s v=(1 0) 00000100:00100000:2.0:1783520936.194041:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36486 00010000:00010000:0.0:1783520936.194043:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x84e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.194050:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026838 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.194058:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026838 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x84e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.194069:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000054c81770/0xd5401cbabc021f28 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x851:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2634 expref: 181 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.194073:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.194083:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507acea00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993261568:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.194093:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.194098:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026846 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x851:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.194103:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+180:2389:x1870153002473344:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 113us (212us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.194112:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026846 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x851:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.194118:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36488 00010000:00010000:2.0:1783520936.194119:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02682a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.194130:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02682a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.194131:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026846 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x851:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.194139:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002473472 00010000:00010000:2.0:1783520936.194141:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02682a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.194141:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026846 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x851:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.194144:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002473472 00010000:00010000:2.0:1783520936.194154:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02682a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x84d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.194154:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026846 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x851:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.194163:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007d2d8f28/0xd5401cbabc021f36 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x853:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2650 expref: 180 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.194174:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520936.194178:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520936.194186:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1783520936.194191:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02682a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x84d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.194195:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.194203:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.194205:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026846 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x851:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.194207:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02682a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x84d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.194227:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.194228:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02682a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x84d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.194235:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026846 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x851:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.194244:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02682a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x84d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.194246:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36489 00010000:00010000:0.0:1783520936.194247:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026846 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x851:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.194263:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515474380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+180:2388:x1870153002473472:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.194267:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02682a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x84d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.194279:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+825:3627:x1870152993260160:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4242us (4324us total) trans 4294976393 rc 0/0 00010000:00010000:0.0:1783520936.194281:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026846 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x851:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.194285:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.194288:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44892 00010000:00010000:1.0:1783520936.194288:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.194291:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.194298:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc026846 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x851:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.194309:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02684d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x852:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.194317:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.194318:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.194319:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02684d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x852:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.194321:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.194326:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.194329:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007d2d8f28/0xd5401cbabc021f36 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x853:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2650 expref: 180 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.194335:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02684d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x852:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.194348:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02684d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x852:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.194349:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007d2d8f28/0xd5401cbabc021f36 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x853:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2650 expref: 180 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.194382:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b3800/2, rc 0, transno 4294976393 00000100:00100000:1.0:1783520936.194385:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635784850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+825:3627:x1870152993261440:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3578us (3675us total) trans 4294976394 rc 0/0 00010000:00010000:2.0:1783520936.194390:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.194392:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44896 00000100:00100000:0.0:1783520936.194397:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993260160:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.194416:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026831 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.194432:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026831 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.194443:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x853:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.194445:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026831 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.194449:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x853:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.194459:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026854 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x853:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.194461:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026831 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x84f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.194464:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507acc700/2, rc 0, transno 4294976394 00000100:00100000:2.0:1783520936.194468:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ae5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+825:3628:x1870152993261696:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3367us (3428us total) trans 4294976395 rc 0/0 00000100:00100000:0.0:1783520936.194482:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507acc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993261440:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.194483:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44897 00010000:00010000:3.0:1783520936.194484:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026854 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x853:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.194503:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1783520936.194517:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ae4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993262208:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.194520:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.194528:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02684d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x852:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.194534:0:20415:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007d2d8f28/0xd5401cbabc021f36 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x853:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2650 expref: 180 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.194540:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb600 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520936.194543:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ae7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993262336:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.194547:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.194548:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02684d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x852:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.194554:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515476d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993262080:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.194567:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02684d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x852:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.194572:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026854 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x853:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.194580:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02684d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x852:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.194586:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.194589:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520936.194593:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.194596:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026831 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x84f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.194617:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc02684d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x852:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.194618:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.194625:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026831 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x84f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.194641:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026831 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x84f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.194643:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.194647:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520936.194647:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507acf480/2, rc 0, transno 4294976395 00010000:00010000:3.0:1783520936.194653:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.194656:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026831 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x84f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.194661:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+825:3625:x1870152993260672:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4546us (4640us total) trans 4294976396 rc 0/0 00000100:00100000:0.0:1783520936.194664:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507acf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993261696:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.194667:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026831 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x84f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.194671:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44893 00010000:00010000:2.0:1783520936.194688:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.194691:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.194693:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520936.194707:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ad0700/2, rc 0, transno 4294976396 00000100:00100000:2.0:1783520936.194708:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993262208 00000100:00100000:2.0:1783520936.194712:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993262208 00000100:00100000:0.0:1783520936.194721:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507acea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993262464:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.194726:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993260672:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.194737:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.194737:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993262336 00000100:00100000:2.0:1783520936.194743:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993262336 00000100:00100000:1.0:1783520936.194748:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44899 00000100:00100000:1.0:1783520936.194755:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+826:3625:x1870152993262208:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.194768:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44900 00000100:00100000:1.0:1783520936.194771:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507acd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993262592:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.194777:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515474380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+179:2388:x1870153002473472:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 515us (679us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.194777:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ae4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+826:3627:x1870152993262336:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.194793:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36489 00002000:00080000:2.0:1783520936.194798:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2133 count 1 00000100:00100000:1.0:1783520936.194805:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.194812:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02685b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x855:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.194824:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000eb756c33/0xd5401cbabc021f44 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x855:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f266c expref: 178 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.194832:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x855:0x0].0x0 since=38s v=(0 0) 00000100:00100000:0.0:1783520936.194837:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.194839:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x855:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.194847:0:8408:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000b160cc6/0xd5401cbabc022a3b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x84e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f312b expref: 178 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.194847:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02685b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x855:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.194848:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2132 count 1 00010000:00010000:1.0:1783520936.194859:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026862 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x854:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.194880:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503371c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993261312:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.194880:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d9885d5a/0xd5401cbabc021f3d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x854:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f265e expref: 177 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.194885:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000eb756c33/0xd5401cbabc021f44 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x855:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f266c expref: 177 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.194894:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x854:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.194896:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x854:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.194899:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000eb756c33/0xd5401cbabc021f44 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x855:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f266c expref: 177 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.194901:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026862 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x854:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.194904:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026838 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.194917:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515476d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993262720:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.194920:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026838 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.194925:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d9885d5a/0xd5401cbabc021f3d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x854:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f265e expref: 177 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.194926:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02683f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x850:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.194936:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026838 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.194952:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02683f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x850:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.194955:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1480 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.194956:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d9885d5a/0xd5401cbabc021f3d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x854:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f265e expref: 177 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.194969:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02683f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x850:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.194975:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b04e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993262848:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.194977:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02683f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x850:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.194999:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.195005:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.195011:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02683f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x850:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.195012:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026838 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.195030:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993262592 00010000:00010000:1.0:1783520936.195030:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.195034:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026838 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x84e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.195035:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993262592 00010000:00010000:1.0:1783520936.195035:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02683f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x850:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.195051:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02683f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x850:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.195060:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993262464 00000100:00100000:2.0:1783520936.195063:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993262464 00010000:00010000:1.0:1783520936.195063:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02683f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x850:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.195069:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+829:3625:x1870152993260800:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4872us (4932us total) trans 4294976397 rc 0/0 00010000:00010000:1.0:1783520936.195073:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc02683f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x850:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.195079:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44895 00000100:00100000:2.0:1783520936.195088:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44901 00000100:00100000:2.0:1783520936.195098:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+828:3625:x1870152993262592:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.195106:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.195106:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44902 00002000:00080000:2.0:1783520936.195110:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2135 count 1 00010000:00010000:1.0:1783520936.195110:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.195115:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026838 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x84e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.195118:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.195123:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026869 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x857:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.195123:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.195126:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026838 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x84e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.195131:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507acf100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+828:3628:x1870152993262464:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.195138:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003278a8c7/0xd5401cbabc021f52 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x857:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2688 expref: 177 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.195143:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026838 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x84e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.195146:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x857:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.195150:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x857:0x0].0x0 since=0s v=(0 0) 00002000:00080000:0.0:1783520936.195151:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2134 count 1 00010000:00010000:2.0:1783520936.195154:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026869 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x857:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.195154:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026838 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x84e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.195166:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026838 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x84e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.195172:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026870 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x856:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.195181:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003278a8c7/0xd5401cbabc021f52 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x857:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2688 expref: 177 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.195181:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.195184:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.195187:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.195188:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f55dd0f0/0xd5401cbabc02292a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x856:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f301a expref: 177 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.195195:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003278a8c7/0xd5401cbabc021f52 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x857:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2688 expref: 177 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.195202:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x856:0x0].0x0 since=37s v=(0 0) 00000004:00080000:1.0:1783520936.195208:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ad1c00/2, rc 0, transno 4294976397 00010000:00010000:0.0:1783520936.195212:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x856:0x0].0x0 since=0s v=(1 0) 00000100:00100000:2.0:1783520936.195215:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515475500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993262976:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.195219:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026870 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x856:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.195220:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993260800:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.195247:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1d80 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.195251:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f55dd0f0/0xd5401cbabc02292a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x856:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f301a expref: 177 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.195266:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f55dd0f0/0xd5401cbabc02292a lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x856:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f301a expref: 177 pid: 8408 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.195269:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026854 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x853:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.195270:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+829:3627:x1870152993260416:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5121us (5200us total) trans 4294976398 rc 0/0 00000100:00100000:1.0:1783520936.195271:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1783520936.195281:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514dd4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993263232:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.195292:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44894 00010000:00010000:2.0:1783520936.195294:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026854 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x853:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.195301:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516310700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993263104:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.195311:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026854 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x853:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.195312:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.195319:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026854 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x853:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.195334:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502251500/2, rc 0, transno 4294976398 00010000:00010000:2.0:1783520936.195337:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.195343:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993263232 00000100:00100000:1.0:1783520936.195349:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993263232 00000100:00100000:0.0:1783520936.195352:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502251500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993260416:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.195422:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1783520936.195432:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44903 00000100:00100000:0.0:1783520936.195433:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514dd4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993263360:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.195442:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514dd6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+830:3625:x1870152993263232:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.195453:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.195460:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026854 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x853:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.195466:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+830:3625:x1870152993261952:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2630us (2777us total) trans 4294976399 rc 0/0 00000004:00080000:1.0:1783520936.195467:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ae6d80/2, rc 0, transno 4294976399 00000100:00100000:0.0:1783520936.195472:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.195479:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ae6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993261952:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.195482:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.195500:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026854 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x853:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.195501:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44898 00010000:00010000:3.0:1783520936.195514:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026854 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x853:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.195523:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993263360 00000100:00100000:2.0:1783520936.195527:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993263360 00010000:00010000:3.0:1783520936.195530:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026854 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x853:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.195539:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026854 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x853:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.195541:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514dd4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993263488:0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.195546:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2136 count 1 00000100:00100000:2.0:1783520936.195548:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44904 00000100:00100000:2.0:1783520936.195560:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516311880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+830:3628:x1870152993263360:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.195563:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026877 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x858:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.195566:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00002000:00080000:2.0:1783520936.195571:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2137 count 1 00010000:00010000:3.0:1783520936.195572:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.195577:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.195578:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000138081c5/0xd5401cbabc021f59 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x858:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2696 expref: 177 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.195582:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.195583:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02687e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x859:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.195591:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x858:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.195595:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x858:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.195596:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008bc8bf3b/0xd5401cbabc021f60 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x859:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f26a4 expref: 177 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.195599:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026877 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x858:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.195604:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x859:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.195606:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x859:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.195610:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02687e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x859:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.195614:0:9151:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000faf075ec/0xd5401cbabc021ed4 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x84c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f25ee expref: 177 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.195620:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000138081c5/0xd5401cbabc021f59 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x858:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2696 expref: 177 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.195634:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008bc8bf3b/0xd5401cbabc021f60 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x859:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26a4 expref: 176 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.195637:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503370380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993260544:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.195639:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000138081c5/0xd5401cbabc021f59 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x858:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2696 expref: 176 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.195649:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80a80 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.195652:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008bc8bf3b/0xd5401cbabc021f60 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x859:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26a4 expref: 176 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.195655:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514dd6680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993263616:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.195662:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026823 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.195663:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515474e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993263744:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.195678:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026823 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.195680:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.195687:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1cc0 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.195690:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026823 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.195699:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026823 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x84c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.195700:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470780 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.195711:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026823 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x84c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.195714:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993263488 00000100:00100000:1.0:1783520936.195719:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993263488 00010000:00010000:3.0:1783520936.195735:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.195740:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44905 00000100:00100000:1.0:1783520936.195748:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516310000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+831:3628:x1870152993263488:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.195759:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2138 count 1 00010000:00010000:2.0:1783520936.195763:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.195768:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026823 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x84c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.195770:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026885 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x85a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.195776:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.195779:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026823 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x84c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.195782:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000004df6711/0xd5401cbabc021f67 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x85a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f26b2 expref: 176 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.195789:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026823 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x84c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.195796:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x85a:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.195799:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x85a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.195801:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026823 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x84c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.195803:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026885 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x85a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.195809:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026823 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x84c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.195820:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000004df6711/0xd5401cbabc021f67 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x85a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26b2 expref: 176 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.195825:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.195826:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.195828:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.195830:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000004df6711/0xd5401cbabc021f67 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x85a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26b2 expref: 176 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.195839:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514dd7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993263872:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.195868:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124706c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.196004:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515474700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+832:3625:x1870152993260032:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6206us (6446us total) trans 4294976400 rc 0/0 00000100:00100000:3.0:1783520936.196013:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44891 00000004:00080000:1.0:1783520936.196031:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515477800/2, rc 0, transno 4294976400 00000100:00100000:1.0:1783520936.196039:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515477800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993260032:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.196072:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514dd5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993264000:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.196090:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.196124:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993264000 00000100:00100000:2.0:1783520936.196129:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993264000 00000100:00100000:2.0:1783520936.196154:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44906 00000100:00100000:2.0:1783520936.196161:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514dd7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+832:3625:x1870152993264000:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.196172:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2139 count 1 00010000:00010000:2.0:1783520936.196187:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02688c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x85b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.196205:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003f3aa56c/0xd5401cbabc021fa6 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x85b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f26c0 expref: 176 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.196213:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x85b:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.196216:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x85b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.196220:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02688c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x85b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.196239:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003f3aa56c/0xd5401cbabc021fa6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x85b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26c0 expref: 176 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.196248:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003f3aa56c/0xd5401cbabc021fa6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x85b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26c0 expref: 176 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.196257:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515475880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993264128:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.196284:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1000 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.196345:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515476d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993262720:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.196377:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02685b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x855:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.197655:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.197693:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002473600 00000100:00100000:1.0:1783520936.197699:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002473600 00000100:00100000:0.0:1783520936.197705:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.197728:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000eb756c33/0xd5401cbabc021f44 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x855:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f266c expref: 177 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.197732:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515475500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993262976:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.197759:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026869 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x857:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.197762:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002473728 00000100:00100000:1.0:1783520936.197767:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002473728 00010000:00010000:1.0:1783520936.197786:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003278a8c7/0xd5401cbabc021f52 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x857:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2688 expref: 178 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.197830:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36490 00000100:00100000:2.0:1783520936.197834:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514dd6680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993263616:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.197847:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f5850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+178:2388:x1870153002473600:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.197856:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026877 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x858:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.197859:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.197860:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36491 00010000:00010000:1.0:1783520936.197870:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000eb756c33/0xd5401cbabc021f44 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x855:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f266c expref: 178 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.197875:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516312a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+178:2390:x1870153002473728:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.197889:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.197892:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.197895:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000eb756c33/0xd5401cbabc021f44 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x855:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f266c expref: 178 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.197898:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003278a8c7/0xd5401cbabc021f52 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x857:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2688 expref: 178 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.197901:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515474e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993263744:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.197921:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.197926:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003278a8c7/0xd5401cbabc021f52 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x857:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2688 expref: 178 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.197927:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02687e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x859:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.197931:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000eb756c33/0xd5401cbabc021f44 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x855:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f266c expref: 178 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.197948:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.197962:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003278a8c7/0xd5401cbabc021f52 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x857:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2688 expref: 177 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.197965:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x855:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.197974:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x855:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.197983:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.197984:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02685b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x855:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.197985:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x857:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.197991:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x857:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.198001:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026869 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x857:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.198001:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002473856 00010000:00010000:1.0:1783520936.198005:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02685b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x855:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.198006:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002473856 00010000:00010000:0.0:1783520936.198022:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000004df6711/0xd5401cbabc021f67 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x85a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26b2 expref: 177 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.198037:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002474112 00000100:00100000:0.0:1783520936.198038:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002474112 00010000:00010000:1.0:1783520936.198045:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.198048:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000138081c5/0xd5401cbabc021f59 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x858:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2696 expref: 178 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.198059:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002473984 00000100:00100000:0.0:1783520936.198060:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002473984 00010000:00010000:2.0:1783520936.198069:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02685b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x855:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.198075:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026869 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x857:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.198076:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f55dd0f0/0xd5401cbabc02292a lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x856:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f301a expref: 179 pid: 8408 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.198087:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f5850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+179:2388:x1870153002473600:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 245us (430us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.198089:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02685b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x855:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.198115:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002474240 00000100:00100000:1.0:1783520936.198116:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36490 00010000:00010000:2.0:1783520936.198117:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02685b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x855:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.198119:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002474240 00010000:00010000:3.0:1783520936.198121:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.198131:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02685b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x855:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.198136:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008bc8bf3b/0xd5401cbabc021f60 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x859:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26a4 expref: 179 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.198154:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516312a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+179:2390:x1870153002473728:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 280us (448us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.198162:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.198168:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36492 00000100:00100000:1.0:1783520936.198177:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36493 00010000:00010000:0.0:1783520936.198179:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:3.0:1783520936.198185:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36491 00010000:00010000:0.0:1783520936.198187:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02685b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x855:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.198190:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516312680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+179:2388:x1870153002474112:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.198203:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.198204:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.198211:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02685b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x855:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.198214:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36494 00010000:00010000:1.0:1783520936.198214:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000138081c5/0xd5401cbabc021f59 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x858:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2696 expref: 178 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.198231:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516311c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+178:2387:x1870153002473984:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.198245:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02685b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x855:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.198247:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000138081c5/0xd5401cbabc021f59 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x858:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2696 expref: 178 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.198267:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02685b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x855:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.198274:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000138081c5/0xd5401cbabc021f59 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x858:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2696 expref: 178 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.198279:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc02685b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x855:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.198284:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ae4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+833:3627:x1870152993262336:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3505us (3695us total) trans 4294976401 rc 0/0 00010000:00010000:3.0:1783520936.198293:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.198299:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x858:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.198303:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f55dd0f0/0xd5401cbabc02292a lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x856:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f301a expref: 177 pid: 8408 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.198308:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x858:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.198317:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44900 00010000:00010000:0.0:1783520936.198317:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.198320:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026877 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x858:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.198321:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.198327:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.198332:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f55dd0f0/0xd5401cbabc02292a lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x856:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f301a expref: 177 pid: 8408 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.198335:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026877 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x858:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.198350:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026869 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x857:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.198380:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ae7b80/2, rc 0, transno 4294976401 00010000:00010000:2.0:1783520936.198391:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026869 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x857:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.198392:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.198395:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x856:0x0].0x0 since=0s v=(1 0) 00000100:00100000:0.0:1783520936.198401:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ae7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993262336:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.198404:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x856:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.198416:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026870 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x856:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.198417:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026869 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x857:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.198425:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516313b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+177:2388:x1870153002473856:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.198437:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026869 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x857:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.198438:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.198445:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000004df6711/0xd5401cbabc021f67 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x85a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26b2 expref: 177 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.198449:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:0.0:1783520936.198460:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000004df6711/0xd5401cbabc021f67 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x85a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26b2 expref: 177 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.198465:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514dd6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993264256:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.198467:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.198472:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x85a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.198476:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x85a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.198480:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.198480:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026885 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.198501:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026869 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x857:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.198508:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026885 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x85a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.198514:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026870 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x856:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.198529:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.198536:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026869 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x857:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.198539:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.198549:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.198549:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026869 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x857:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.198563:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516313b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+177:2388:x1870153002473856:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 387us (675us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.198566:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026869 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x857:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.198570:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+833:3625:x1870152993262592:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3471us (3763us total) trans 4294976402 rc 0/0 00000100:00100000:0.0:1783520936.198575:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36492 00010000:00010000:1.0:1783520936.198588:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026869 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x857:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.198589:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44901 00000100:00100000:0.0:1783520936.198596:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36495 00000100:00100000:0.0:1783520936.198607:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c5bb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+176:2387:x1870153002474240:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.198614:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.198617:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516311c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+176:2387:x1870153002473984:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 388us (672us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.198620:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026877 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x858:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.198621:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008bc8bf3b/0xd5401cbabc021f60 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x859:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26a4 expref: 176 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.198624:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.198628:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.198633:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.198637:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36494 00010000:00010000:2.0:1783520936.198649:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026877 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x858:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.198654:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008bc8bf3b/0xd5401cbabc021f60 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x859:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26a4 expref: 175 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.198667:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026877 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x858:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.198673:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.198678:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008bc8bf3b/0xd5401cbabc021f60 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x859:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26a4 expref: 175 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.198698:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.198704:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026877 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x858:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.198704:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x859:0x0].0x0 since=0s v=(0 0) 00000004:00080000:1.0:1783520936.198710:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507acd500/2, rc 0, transno 4294976402 00010000:00010000:0.0:1783520936.198710:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x859:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.198719:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02687e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x859:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.198727:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507acd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993262592:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.198733:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02687e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x859:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.198736:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.198749:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.198756:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026877 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x858:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.198756:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.198769:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.198774:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026877 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x858:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.198778:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c5bb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+174:2387:x1870153002474240:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 173us (799us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.198792:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36495 00000100:00100000:1.0:1783520936.198802:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516312680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+174:2388:x1870153002474112:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 615us (883us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.198803:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026877 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x858:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.198810:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002474368 00000100:00100000:0.0:1783520936.198817:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002474368 00000100:00100000:1.0:1783520936.198821:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36493 00000100:00100000:2.0:1783520936.198827:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514dd6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+833:3625:x1870152993263232:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3385us (3512us total) trans 4294976403 rc 0/0 00010000:00010000:3.0:1783520936.198828:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026877 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x858:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.198836:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d9885d5a/0xd5401cbabc021f3d lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x854:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f265e expref: 173 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.198847:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44903 00010000:00010000:3.0:1783520936.198853:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026877 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x858:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.198856:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514dd4e00/2, rc 0, transno 4294976403 00000100:00100000:2.0:1783520936.198875:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993264256 00000100:00100000:0.0:1783520936.198875:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36496 00000100:00100000:1.0:1783520936.198876:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514dd4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993263232:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.198878:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.198882:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993264256 00010000:00010000:3.0:1783520936.198883:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.198886:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503370380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+173:2390:x1870153002474368:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.198889:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.198895:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.198900:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d9885d5a/0xd5401cbabc021f3d lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x854:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f265e expref: 173 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.198909:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d9885d5a/0xd5401cbabc021f3d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x854:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f265e expref: 173 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.198921:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44907 00010000:00010000:0.0:1783520936.198921:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x854:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.198926:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x854:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.198931:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026862 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x854:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.198937:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514dd6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+833:3625:x1870152993264256:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.198940:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026862 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x854:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.198955:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00002000:00080000:2.0:1783520936.198960:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2140 count 1 00010000:00010000:1.0:1783520936.198960:0:35275:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d9885d5a/0xd5401cbabc021f3d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x854:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f265e expref: 173 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.198970:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.198980:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503370380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+172:2390:x1870153002474368:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 95us (286us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.198981:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026893 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x85c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.198986:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b04e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993262848:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.198993:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002474496 00000100:00100000:0.0:1783520936.198999:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36496 00000100:00100000:3.0:1783520936.199000:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002474496 00010000:00010000:1.0:1783520936.199008:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026862 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x854:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.199010:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000030579f03/0xd5401cbabc021fbb lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x85c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f26ce expref: 173 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.199025:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003f3aa56c/0xd5401cbabc021fa6 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x85b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26c0 expref: 172 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.199030:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435080d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993264512:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.199032:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026862 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x854:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.199038:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x85c:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.199048:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x85c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.199059:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026893 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x85c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.199060:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026862 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x854:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.199066:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36497 00000100:00100000:0.0:1783520936.199074:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.199080:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f0050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+172:2388:x1870153002474496:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.199090:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026862 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x854:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.199093:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.199103:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003f3aa56c/0xd5401cbabc021fa6 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x85b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26c0 expref: 172 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.199106:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000030579f03/0xd5401cbabc021fbb lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x85c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26ce expref: 172 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.199106:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026862 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x854:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.199106:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1783520936.199117:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435080d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993264384:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.199134:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003f3aa56c/0xd5401cbabc021fa6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x85b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26c0 expref: 172 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.199151:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.199156:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.199161:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x85b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.199166:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x85b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.199166:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000030579f03/0xd5401cbabc021fbb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x85c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26ce expref: 172 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.199174:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02688c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.199187:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02688c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x85b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.199192:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515474e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993264640:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.199201:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.199208:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.199215:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026862 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x854:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.199220:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f55dd0f0/0xd5401cbabc02292a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x856:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f301a expref: 172 pid: 8408 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.199230:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbbe00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:3.0:1783520936.199231:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f0050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+172:2388:x1870153002474496:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 151us (261us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.199238:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026862 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x854:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.199247:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516310700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993263104:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.199249:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36497 00000100:00100000:1.0:1783520936.199253:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+834:3625:x1870152993262208:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4495us (4704us total) trans 4294976404 rc 0/0 00010000:00010000:2.0:1783520936.199257:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026862 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x854:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.199269:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44899 00010000:00010000:0.0:1783520936.199272:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026870 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x856:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.199278:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026862 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x854:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.199285:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026870 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x856:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.199291:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026862 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x854:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.199294:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993264512 00010000:00010000:0.0:1783520936.199298:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026870 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x856:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.199300:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993264512 00000004:00080000:1.0:1783520936.199306:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ae4700/2, rc 0, transno 4294976404 00000100:00100000:3.0:1783520936.199318:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993264384 00000100:00100000:3.0:1783520936.199322:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993264384 00000100:00100000:1.0:1783520936.199323:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ae4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993262208:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.199326:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.199327:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026870 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x856:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.199332:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.199338:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.199344:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026870 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x856:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.199345:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44908 00000100:00100000:3.0:1783520936.199355:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c99c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+835:3628:x1870152993264512:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.199383:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44909 00002000:00080000:3.0:1783520936.199384:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2142 count 1 00010000:00010000:0.0:1783520936.199390:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.199397:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02689a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x85e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.199397:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c9b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+835:3628:x1870152993264384:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.199403:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.199411:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026870 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x856:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.199419:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000090810493/0xd5401cbabc022b84 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x85e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3274 expref: 170 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.199419:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2141 count 1 00000100:00100000:1.0:1783520936.199422:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993264768:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.199428:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.199435:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0268a1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x85d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.199436:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026870 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x856:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.199438:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x85e:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.199448:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x85e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.199459:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02689a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x85e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.199463:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.199466:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026870 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x856:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.199468:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d714b763/0xd5401cbabc021fc9 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x85d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f26dc expref: 170 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.199504:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x85d:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.199509:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x85d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.199510:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026870 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x856:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.199514:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0268a1 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x85d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.199521:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000090810493/0xd5401cbabc022b84 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x85e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3274 expref: 170 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.199527:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026870 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x856:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.199552:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000090810493/0xd5401cbabc022b84 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x85e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3274 expref: 170 pid: 12734 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.199560:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d714b763/0xd5401cbabc021fc9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x85d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26dc expref: 170 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.199564:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.199565:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503370380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993264896:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.199569:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.199574:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.199584:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d714b763/0xd5401cbabc021fc9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x85d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26dc expref: 170 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.199598:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515475500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993265024:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.199617:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57d80 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.199633:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1180 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.199638:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993264768 00000100:00100000:3.0:1783520936.199644:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993264768 00010000:00010000:2.0:1783520936.199675:0:178496:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003f3aa56c/0xd5401cbabc021fa6 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x85b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26c0 expref: 170 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.199687:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507acf100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+837:3628:x1870152993262464:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4571us (4855us total) trans 4294976405 rc 0/0 00000100:00100000:2.0:1783520936.199693:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515475880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993264128:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.199694:0:35274:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000004df6711/0xd5401cbabc021f67 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x85a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26b2 expref: 169 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.199699:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44902 00000100:00100000:1.0:1783520936.199719:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514dd7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993263872:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.199720:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02688c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.199732:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02688c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.199741:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026885 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.199742:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02688c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.199743:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507acea00/2, rc 0, transno 4294976405 00000100:00100000:3.0:1783520936.199753:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44910 00000100:00100000:0.0:1783520936.199763:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507acea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993262464:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.199770:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+836:3625:x1870152993264768:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.199772:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02688c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.199784:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02688c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x85b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.199788:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2143 count 1 00010000:00010000:2.0:1783520936.199805:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.199821:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520936.199824:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:0.0:1783520936.199828:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02688c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x85b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.199835:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993265152:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.199839:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.199845:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02688c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x85b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.199858:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02688c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x85b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.199868:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.199874:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02688c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x85b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.199875:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0268a8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x85f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.199895:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc02688c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x85b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.199897:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514dd7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+837:3625:x1870152993264000:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3734us (3804us total) trans 4294976406 rc 0/0 00010000:00010000:3.0:1783520936.199903:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000dae94219/0xd5401cbabc021fec lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x85f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f26f8 expref: 168 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.199903:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514dd5180/2, rc 0, transno 4294976406 00000100:00100000:1.0:1783520936.199918:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514dd5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993264000:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.199920:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44906 00010000:00010000:3.0:1783520936.199924:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x85f:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.199925:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.199929:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.199931:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x85f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.199935:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.199937:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0268a8 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x85f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.199944:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993265152 00010000:00010000:0.0:1783520936.199947:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026885 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.199950:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993265152 00010000:00010000:0.0:1783520936.199966:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026885 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.199973:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993265280:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.199976:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026885 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.199981:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000dae94219/0xd5401cbabc021fec lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x85f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26f8 expref: 168 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.199984:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44911 00000100:00100000:2.0:1783520936.199999:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+837:3625:x1870152993265152:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.200001:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026885 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x85a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.200008:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000dae94219/0xd5401cbabc021fec lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x85f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26f8 expref: 168 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.200009:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00080000:2.0:1783520936.200021:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2144 count 1 00000100:00100000:3.0:1783520936.200035:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503371c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993265408:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.200038:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0268af lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x860:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.200042:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.200050:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993265280 00010000:00010000:3.0:1783520936.200053:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520936.200056:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993265280 00010000:00010000:3.0:1783520936.200062:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026885 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x85a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.200070:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000633f1687/0xd5401cbabc021ffa lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x860:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2706 expref: 168 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.200071:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.200077:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026885 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x85a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.200080:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44912 00010000:00010000:2.0:1783520936.200092:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x860:0x0].0x0 since=38s v=(0 0) 00000100:00100000:1.0:1783520936.200094:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+839:3627:x1870152993265280:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.200099:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026885 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x85a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.200101:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x860:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.200110:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0268af lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x860:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.200113:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2145 count 1 00010000:00010000:3.0:1783520936.200119:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026885 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x85a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.200133:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026885 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x85a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.200133:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0268b6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x861:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.200140:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516310000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+839:3628:x1870152993263488:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4390us (4559us total) trans 4294976407 rc 0/0 00010000:00010000:2.0:1783520936.200143:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000633f1687/0xd5401cbabc021ffa lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x860:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2706 expref: 168 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.200162:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44905 00010000:00010000:1.0:1783520936.200167:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000047863a18/0xd5401cbabc02200f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x861:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2714 expref: 168 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.200174:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000633f1687/0xd5401cbabc021ffa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x860:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2706 expref: 168 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.200194:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x861:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.200200:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x861:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.200201:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515475880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993265536:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.200207:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0268b6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x861:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.200214:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514dd4000/2, rc 0, transno 4294976407 00000100:00100000:0.0:1783520936.200230:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514dd4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993263488:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.200239:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1a80 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.200266:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.200271:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:3.0:1783520936.200272:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.200277:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02687e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x859:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.200278:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.200283:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993265664:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.200293:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02687e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x859:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.200303:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02687e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x859:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.200315:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a87a80 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.200317:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02687e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x859:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.200330:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.200345:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.200351:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993265664 00000100:00100000:0.0:1783520936.200379:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993265664 00010000:00010000:3.0:1783520936.200385:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.200392:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02687e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x859:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.200402:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.200404:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000047863a18/0xd5401cbabc02200f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x861:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2714 expref: 168 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.200409:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02687e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x859:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.200413:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44913 00010000:00010000:1.0:1783520936.200426:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000047863a18/0xd5401cbabc02200f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x861:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2714 expref: 168 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.200428:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514dd7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+840:3627:x1870152993265664:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.200446:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993265792:192.168.204.33@tcp:104: 00002000:00080000:0.0:1783520936.200449:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2146 count 1 00010000:00010000:0.0:1783520936.200464:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0268bd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x862:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.200477:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516311880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+840:3628:x1870152993263360:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4917us (5007us total) trans 4294976408 rc 0/0 00010000:00010000:0.0:1783520936.200479:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001293d8a6/0xd5401cbabc022016 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x862:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2722 expref: 168 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.200484:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514dd4700/2, rc 0, transno 4294976408 00000100:00100000:2.0:1783520936.200505:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44904 00000100:00100000:1.0:1783520936.200512:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514dd4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993263360:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.200513:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02687e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x859:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.200517:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x862:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.200526:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x862:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.200533:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0268bd lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x862:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.200536:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02687e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x859:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.200554:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02687e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x859:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.200567:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:0.0:1783520936.200571:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001293d8a6/0xd5401cbabc022016 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x862:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2722 expref: 168 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.200573:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1780 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.200579:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516358700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993265920:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.200593:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001293d8a6/0xd5401cbabc022016 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x862:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2722 expref: 168 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.200618:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502c99500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993266048:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.200619:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.200622:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.200627:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.200631:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.200656:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80300 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.200663:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993265920 00000100:00100000:2.0:1783520936.200668:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993265920 00000100:00100000:2.0:1783520936.200702:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44914 00000100:00100000:1.0:1783520936.200705:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124706c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.200712:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351635b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+840:3626:x1870152993265920:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.200725:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2147 count 1 00010000:00010000:2.0:1783520936.200738:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0268c4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x863:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.200750:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006805e60c/0xd5401cbabc022024 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x863:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2730 expref: 168 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.200758:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x863:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.200762:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x863:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.200766:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0268c4 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x863:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.200790:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006805e60c/0xd5401cbabc022024 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x863:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2730 expref: 168 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.200801:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006805e60c/0xd5401cbabc022024 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x863:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2730 expref: 168 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.200812:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515476d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993266176:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.200850:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b16c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.200875:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503370380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993264896:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.200890:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02689a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x85e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.202121:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.202175:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002474624 00000100:00100000:0.0:1783520936.202181:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002474624 00010000:00010000:0.0:1783520936.202206:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000090810493/0xd5401cbabc022b84 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x85e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3274 expref: 169 pid: 12734 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.202213:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515475500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993265024:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.202232:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515475880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993265536:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.202241:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0268a1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x85d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.202248:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36498 00000100:00100000:0.0:1783520936.202264:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435038ec380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+169:2387:x1870153002474624:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.202274:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.202281:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000090810493/0xd5401cbabc022b84 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x85e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3274 expref: 169 pid: 12734 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.202284:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515474e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993264640:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.202303:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026893 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x85c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.202308:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000090810493/0xd5401cbabc022b84 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x85e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3274 expref: 169 pid: 12734 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.202340:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0268af lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x860:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.202348:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000090810493/0xd5401cbabc022b84 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x85e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3274 expref: 169 pid: 12734 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.202388:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x85e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.202392:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x85e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.202397:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02689a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.202403:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503371c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993265408:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.202412:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02689a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x85e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.202435:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0268a8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x85f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.202437:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.202476:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435038ec380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+168:2387:x1870153002474624:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 216us (356us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.202485:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36498 00000100:00100000:2.0:1783520936.202499:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515476d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993266176:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.202520:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02689a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.202523:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0268c4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x863:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.202540:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502c99500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993266048:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.202556:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0268bd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x862:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.202620:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.202650:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.202650:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002474752 00000100:00100000:0.0:1783520936.202655:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002474752 00010000:00010000:0.0:1783520936.202670:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d714b763/0xd5401cbabc021fc9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x85d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26dc expref: 168 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.202679:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02689a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.202692:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002475008 00010000:00010000:3.0:1783520936.202693:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02689a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.202697:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002475008 00010000:00010000:3.0:1783520936.202706:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02689a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x85e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.202713:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000dae94219/0xd5401cbabc021fec lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x85f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26f8 expref: 169 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.202742:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36499 00000100:00100000:2.0:1783520936.202753:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993265792:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.202754:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435038ed180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+169:2388:x1870153002474752:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.202768:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.202771:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36500 00010000:00010000:2.0:1783520936.202773:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0268b6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x861:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.202776:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d714b763/0xd5401cbabc021fc9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x85d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26dc expref: 169 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.202788:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435038eea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+169:2388:x1870153002475008:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.202800:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.202808:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d714b763/0xd5401cbabc021fc9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x85d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26dc expref: 169 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.202809:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000dae94219/0xd5401cbabc021fec lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x85f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26f8 expref: 169 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.202834:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d714b763/0xd5401cbabc021fc9 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x85d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26dc expref: 169 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.202838:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000dae94219/0xd5401cbabc021fec lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x85f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26f8 expref: 169 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.202850:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x85d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.202854:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x85d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.202861:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000dae94219/0xd5401cbabc021fec lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x85f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26f8 expref: 168 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.202862:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0268a1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.202884:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0268a1 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x85d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.202889:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x85f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.202894:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x85f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.202899:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0268a8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.202906:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.202909:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0268a8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x85f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.202932:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435038ed180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+167:2388:x1870153002474752:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 182us (313us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.202934:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.202941:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36499 00000100:00100000:1.0:1783520936.202964:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435038eea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+166:2388:x1870153002475008:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 180us (314us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.202973:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.202974:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36500 00010000:00010000:0.0:1783520936.202986:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.202992:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02689a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x85e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.203001:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.203005:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02689a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x85e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.203014:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02689a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x85e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.203025:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02689a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x85e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.203033:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02689a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x85e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.203043:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.203054:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.203058:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.203060:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.203080:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.203091:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0268a1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.203101:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002475392 00010000:00010000:2.0:1783520936.203103:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0268a1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.203107:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002475392 00010000:00010000:2.0:1783520936.203116:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0268a1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.203126:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0268a1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x85d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.203130:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006805e60c/0xd5401cbabc022024 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x863:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2730 expref: 166 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.203137:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002474880 00000100:00100000:1.0:1783520936.203145:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002474880 00010000:00010000:2.0:1783520936.203155:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520936.203166:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36501 00010000:00010000:1.0:1783520936.203168:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000633f1687/0xd5401cbabc021ffa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x860:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2706 expref: 167 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.203168:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.203177:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0268a1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x85d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.203180:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+167:2388:x1870153002475392:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.203193:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.203195:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.203198:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0268a1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x85d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.203203:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006805e60c/0xd5401cbabc022024 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x863:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2730 expref: 167 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.203207:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36502 00010000:00010000:0.0:1783520936.203214:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0268a1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x85d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.203222:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503370380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+167:2388:x1870153002474880:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.203225:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006805e60c/0xd5401cbabc022024 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x863:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2730 expref: 167 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.203237:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.203242:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0268a1 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x85d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.203247:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000633f1687/0xd5401cbabc021ffa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x860:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2706 expref: 167 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.203251:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006805e60c/0xd5401cbabc022024 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x863:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2730 expref: 167 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.203258:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c9b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+842:3628:x1870152993264384:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3863us (4103us total) trans 4294976410 rc 0/0 00010000:00010000:0.0:1783520936.203261:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc0268a1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x85d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.203275:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000633f1687/0xd5401cbabc021ffa lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x860:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2706 expref: 167 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.203276:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44909 00010000:00010000:1.0:1783520936.203296:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000633f1687/0xd5401cbabc021ffa lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x860:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2706 expref: 167 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.203306:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.203309:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.203313:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.203322:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x863:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.203323:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x860:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.203331:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x863:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.203337:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0268c4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x863:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.203342:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x860:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.203349:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0268c4 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x863:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.203351:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c99c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+841:3628:x1870152993264512:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3995us (4280us total) trans 4294976409 rc 0/0 00010000:00010000:1.0:1783520936.203354:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0268af lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x860:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.203387:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0268af lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x860:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.203404:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44908 00000004:00080000:1.0:1783520936.203417:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435080d3480/2, rc 0, transno 4294976410 00000100:00100000:1.0:1783520936.203426:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435080d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993264384:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.203430:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000004:00080000:1.0:1783520936.203435:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435080d0380/2, rc 0, transno 4294976409 00000100:00100000:1.0:1783520936.203440:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435080d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993264512:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.203453:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.203463:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+165:2388:x1870153002475392:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 288us (422us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.203472:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36501 00000100:00100000:0.0:1783520936.203481:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503370380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+165:2388:x1870153002474880:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 262us (400us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.203503:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36502 00000100:00100000:3.0:1783520936.203529:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.203538:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435038ece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993266304:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.203552:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435038efb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993266432:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.203572:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0268c4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x863:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.203582:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0268c4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x863:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.203586:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.203598:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0268c4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x863:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.203606:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0268c4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x863:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.203631:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993266304 00000100:00100000:3.0:1783520936.203634:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002475136 00010000:00010000:2.0:1783520936.203634:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.203636:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993266304 00000100:00100000:3.0:1783520936.203641:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002475136 00000100:00100000:1.0:1783520936.203647:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.203654:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520936.203676:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993266432 00000100:00100000:0.0:1783520936.203681:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993266432 00000100:00100000:1.0:1783520936.203695:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44915 00010000:00010000:3.0:1783520936.203704:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0268c4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x863:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.203709:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436342b8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+843:3626:x1870152993266304:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.203718:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44916 00010000:00010000:3.0:1783520936.203726:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520936.203732:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436342b8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+843:3627:x1870152993266432:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.203733:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0268c4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x863:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.203740:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351635b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+843:3626:x1870152993265920:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3028us (3121us total) trans 4294976411 rc 0/0 00002000:00080000:0.0:1783520936.203749:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2149 count 1 00010000:00010000:3.0:1783520936.203750:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0268c4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x863:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.203752:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516358700/2, rc 0, transno 4294976411 00000100:00100000:2.0:1783520936.203754:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44914 00000100:00100000:1.0:1783520936.203766:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516358700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993265920:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.203769:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0268cb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x865:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.203772:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0268c4 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x863:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.203788:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0268c4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x863:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.203789:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006399409c/0xd5401cbabc022047 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x865:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f274c expref: 164 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.203808:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x865:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.203812:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x865:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.203816:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0268cb lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x865:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.203818:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.203821:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:3.0:1783520936.203823:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.203827:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.203833:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94360550f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993266560:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.203846:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006399409c/0xd5401cbabc022047 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x865:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f274c expref: 164 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.203854:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000030579f03/0xd5401cbabc021fbb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x85c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26ce expref: 164 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.203862:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006399409c/0xd5401cbabc022047 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x865:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f274c expref: 164 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.203867:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.203892:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502c99500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993266688:192.168.204.33@tcp:104: 00002000:00080000:1.0:1783520936.203898:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2148 count 1 00000100:00100000:3.0:1783520936.203905:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36503 00000100:00100000:3.0:1783520936.203915:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+164:2388:x1870153002475136:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.203916:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0268d2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x864:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.203927:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.203933:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c39dae6d/0xd5401cbabc02202b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x864:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f273e expref: 164 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.203936:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000030579f03/0xd5401cbabc021fbb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x85c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26ce expref: 164 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.203944:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80300 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.203947:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x864:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.203954:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x864:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.203955:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993266560 00010000:00010000:1.0:1783520936.203959:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0268d2 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x864:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.203961:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993266560 00010000:00010000:3.0:1783520936.203963:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000030579f03/0xd5401cbabc021fbb lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x85c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26ce expref: 164 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.203986:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000030579f03/0xd5401cbabc021fbb lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x85c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f26ce expref: 164 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.203991:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c39dae6d/0xd5401cbabc02202b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x864:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f273e expref: 164 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.203996:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44917 00000100:00100000:2.0:1783520936.204007:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94360550c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+843:3627:x1870152993266560:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.204008:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x85c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.204018:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x85c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.204026:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026893 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.204027:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c39dae6d/0xd5401cbabc02202b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x864:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f273e expref: 163 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.204028:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2150 count 1 00010000:00010000:0.0:1783520936.204043:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0268a8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.204044:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0268d9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x866:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.204049:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94360550d180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993266816:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.204069:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0268a8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.204073:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000132e862b/0xd5401cbabc022a73 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x866:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3163 expref: 163 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.204080:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470900 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.204090:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0268a8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.204091:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x866:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.204098:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x866:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.204105:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0268d9 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x866:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.204108:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0268a8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x85f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.204119:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026893 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x85c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.204132:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000132e862b/0xd5401cbabc022a73 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x866:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3163 expref: 163 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.204137:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.204141:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.204149:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000132e862b/0xd5401cbabc022a73 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x866:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3163 expref: 163 pid: 9151 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.204162:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515476d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993266944:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.204171:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.204197:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002475264 00000100:00100000:2.0:1783520936.204199:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1180 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.204202:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002475264 00010000:00010000:1.0:1783520936.204224:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001293d8a6/0xd5401cbabc022016 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x862:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2722 expref: 164 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.204236:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+844:3625:x1870152993264768:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4471us (4777us total) trans 4294976412 rc 0/0 00000100:00100000:0.0:1783520936.204245:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44910 00000100:00100000:1.0:1783520936.204250:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002475520 00000100:00100000:1.0:1783520936.204256:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002475520 00010000:00010000:3.0:1783520936.204273:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.204274:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000047863a18/0xd5401cbabc02200f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x861:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2714 expref: 165 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.204280:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0268a8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x85f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.204281:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351575c380/2, rc 0, transno 4294976412 00000100:00100000:0.0:1783520936.204295:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993264768:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.204307:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36504 00000100:00100000:1.0:1783520936.204317:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f6c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+165:2390:x1870153002475264:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.204320:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0268af lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x860:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.204327:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.204331:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0268af lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x860:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.204334:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001293d8a6/0xd5401cbabc022016 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x862:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2722 expref: 165 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.204339:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515474e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993267072:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.204354:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0268af lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x860:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.204396:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.204412:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.204412:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001293d8a6/0xd5401cbabc022016 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x862:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2722 expref: 165 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.204415:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0268af lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x860:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.204420:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0268a8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x85f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.204439:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36505 00010000:00010000:3.0:1783520936.204445:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0268a8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x85f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.204447:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001293d8a6/0xd5401cbabc022016 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x862:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2722 expref: 165 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.204447:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.204454:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94360550c000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+165:2390:x1870153002475520:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.204460:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.204466:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0268af lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x860:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.204472:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x862:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.204472:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.204477:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0268af lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x860:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.204478:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x862:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.204484:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.204500:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0268bd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x862:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.204502:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0268af lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x860:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.204506:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000047863a18/0xd5401cbabc02200f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x861:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2714 expref: 164 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.204530:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0268bd lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x862:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.204533:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0268af lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x860:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.204536:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000047863a18/0xd5401cbabc02200f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x861:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2714 expref: 164 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.204557:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005461c69f/0xd5401cbabc0268af lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x860:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.204568:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000047863a18/0xd5401cbabc02200f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x861:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2714 expref: 164 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.204571:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.204578:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.204583:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.204587:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x861:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.204587:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.204594:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x861:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.204600:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0268b6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x861:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.204601:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0268a8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x85f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.204602:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f6c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+163:2390:x1870153002475264:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 286us (463us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.204621:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36504 00010000:00010000:2.0:1783520936.204624:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0268b6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x861:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.204625:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc0268a8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x85f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.204664:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.204664:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.204664:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0268bd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x862:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.204669:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.204674:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.204677:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0268bd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x862:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.204679:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+844:3625:x1870152993265152:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4682us (4810us total) trans 4294976413 rc 0/0 00000100:00100000:2.0:1783520936.204696:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94360550c000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+162:2390:x1870153002475520:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 239us (521us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.204696:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0268bd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x862:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.204697:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44911 00010000:00010000:3.0:1783520936.204702:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.204714:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0268bd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x862:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.204715:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36505 00000100:00100000:3.0:1783520936.204732:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+161:2388:x1870153002475136:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 816us (1202us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.204738:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000004:00080000:0.0:1783520936.204738:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351575ed80/2, rc 0, transno 4294976413 00000100:00100000:3.0:1783520936.204742:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36503 00000100:00100000:2.0:1783520936.204744:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993267072 00000100:00100000:2.0:1783520936.204749:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993267072 00000100:00100000:0.0:1783520936.204760:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993265152:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.204772:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.204778:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0268bd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x862:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.204787:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.204792:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0268bd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x862:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.204800:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0268bd lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x862:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.204803:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515475880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993267200:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.204820:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0268bd lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x862:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.204828:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc0268bd lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x862:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.204842:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.204844:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.204848:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.204851:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.204880:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993267200 00010000:00010000:3.0:1783520936.204883:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026893 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.204884:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993267200 00000100:00100000:0.0:1783520936.204888:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44918 00010000:00010000:3.0:1783520936.204899:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026893 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.204904:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515475500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+846:3625:x1870152993267072:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.204910:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44919 00000100:00100000:2.0:1783520936.204922:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d28a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+846:3626:x1870152993267200:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.204922:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514dd7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+846:3627:x1870152993265664:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4495us (4593us total) trans 4294976414 rc 0/0 00010000:00010000:3.0:1783520936.204926:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026893 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x85c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.204927:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2151 count 1 00002000:00080000:2.0:1783520936.204939:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2152 count 1 00000100:00100000:1.0:1783520936.204941:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44913 00010000:00010000:0.0:1783520936.204942:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0268e0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x867:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.204944:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026893 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x85c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.204955:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0268e7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x868:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.204966:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000065b19b2b/0xd5401cbabc02205c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x867:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2768 expref: 160 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.204971:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351575c000/2, rc 0, transno 4294976414 00010000:00010000:2.0:1783520936.204975:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000033652089/0xd5401cbabc022063 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x868:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2776 expref: 160 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.204977:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.204979:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x867:0x0].0x0 since=38s v=(0 0) 00000100:00100000:1.0:1783520936.204987:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993265664:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.204987:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x867:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.204991:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.204994:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x868:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.204996:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0268e0 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x867:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.205000:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026893 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x85c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.205001:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x868:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.205012:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0268e7 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x868:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.205014:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.205021:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026893 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x85c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.205027:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000065b19b2b/0xd5401cbabc02205c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x867:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2768 expref: 160 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.205035:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026893 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x85c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.205042:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000065b19b2b/0xd5401cbabc02205c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x867:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2768 expref: 160 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.205044:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993267328:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.205052:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026893 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x85c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.205054:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000033652089/0xd5401cbabc022063 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x868:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2776 expref: 160 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.205073:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502c9a680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993267456:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.205078:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026893 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x85c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.205078:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.205084:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000033652089/0xd5401cbabc022063 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x868:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2776 expref: 160 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.205104:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515474380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993267584:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.205111:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993267328 00010000:00010000:3.0:1783520936.205112:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.205116:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.205117:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993267328 00010000:00010000:3.0:1783520936.205122:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.205131:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1300 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.205131:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80780 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.205145:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44920 00000100:00100000:1.0:1783520936.205154:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d2bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+846:3627:x1870152993267328:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.205167:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2153 count 1 00010000:00010000:0.0:1783520936.205175:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0268b6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x861:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.205181:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0268ee lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x869:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.205192:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0268b6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x861:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.205199:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000064bc176d/0xd5401cbabc02206a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x869:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2784 expref: 160 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.205206:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0268b6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x861:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.205208:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x869:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.205216:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x869:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.205225:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0268ee lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x869:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.205227:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0268b6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x861:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.205251:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.205256:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000064bc176d/0xd5401cbabc02206a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x869:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2784 expref: 160 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.205260:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.205267:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0268b6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x861:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.205270:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000064bc176d/0xd5401cbabc02206a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x869:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2784 expref: 160 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.205287:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba800 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520936.205293:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d29880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993267712:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.205293:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0268b6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x861:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.205307:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0268b6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x861:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.205320:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514dd6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+847:3625:x1870152993264256:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6387us (6649us total) trans 4294976415 rc 0/0 00010000:00010000:0.0:1783520936.205321:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0268b6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x861:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.205335:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44907 00000100:00100000:1.0:1783520936.205335:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470a80 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.205336:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc0268b6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x861:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.205341:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514dd6300/2, rc 0, transno 4294976415 00000100:00100000:2.0:1783520936.205354:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514dd6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993264256:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.205383:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.205386:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.205389:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.205427:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993267840:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.205468:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.205499:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+847:3627:x1870152993265280:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5396us (5483us total) trans 4294976416 rc 0/0 00000004:00080000:1.0:1783520936.205509:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351575f800/2, rc 0, transno 4294976416 00000100:00100000:0.0:1783520936.205509:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44912 00000100:00100000:1.0:1783520936.205522:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993265280:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.205531:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993267840 00000100:00100000:2.0:1783520936.205537:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993267840 00000100:00100000:2.0:1783520936.205573:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44921 00000100:00100000:1.0:1783520936.205578:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1783520936.205583:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515474700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+847:3628:x1870152993267840:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.205590:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993267968:0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.205597:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2154 count 1 00010000:00010000:2.0:1783520936.205613:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0268f5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x86a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.205624:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.205634:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000004b91623/0xd5401cbabc022071 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x86a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2792 expref: 160 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.205644:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x86a:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.205648:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x86a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.205653:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0268f5 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x86a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.205659:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993267968 00000100:00100000:0.0:1783520936.205665:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993267968 00010000:00010000:2.0:1783520936.205675:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000004b91623/0xd5401cbabc022071 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x86a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2792 expref: 160 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.205692:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000004b91623/0xd5401cbabc022071 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x86a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2792 expref: 160 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.205702:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44922 00000100:00100000:2.0:1783520936.205707:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515476a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993268096:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.205714:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350622b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+848:3626:x1870152993267968:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.205729:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2155 count 1 00010000:00010000:0.0:1783520936.205742:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0268fc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x86b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.205755:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ebe32b4c/0xd5401cbabc022078 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x86b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f27a0 expref: 160 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.205761:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b19c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.205763:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x86b:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.205768:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x86b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.205773:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0268fc lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x86b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.205795:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ebe32b4c/0xd5401cbabc022078 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x86b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27a0 expref: 160 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.205806:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ebe32b4c/0xd5401cbabc022078 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x86b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27a0 expref: 160 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.205820:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516312a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993268224:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.205824:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.205849:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002475648 00000100:00100000:1.0:1783520936.205854:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002475648 00000100:00100000:0.0:1783520936.205866:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80e40 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.205872:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006399409c/0xd5401cbabc022047 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x865:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f274c expref: 161 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.205901:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36506 00000100:00100000:0.0:1783520936.205901:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502c99500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993266688:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.205913:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+161:2388:x1870153002475648:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.205921:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0268cb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x865:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.205923:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.205931:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006399409c/0xd5401cbabc022047 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x865:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f274c expref: 161 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.205950:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006399409c/0xd5401cbabc022047 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x865:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f274c expref: 161 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.205965:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006399409c/0xd5401cbabc022047 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x865:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f274c expref: 161 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.205976:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x865:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.205979:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x865:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.205983:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0268cb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x865:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.205990:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0268cb lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x865:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.206011:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.206019:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94360550d180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993266816:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.206035:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0268cb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x865:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.206040:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0268d2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x864:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.206046:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+160:2388:x1870153002475648:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 133us (223us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.206052:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0268cb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x865:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.206066:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36506 00010000:00010000:0.0:1783520936.206071:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0268cb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x865:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.206080:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0268cb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x865:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.206100:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515476d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993266944:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.206101:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.206120:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0268d9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x866:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.206183:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436342b8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+849:3627:x1870152993266432:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2455us (2537us total) trans 4294976417 rc 0/0 00000100:00100000:0.0:1783520936.206191:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44916 00000004:00080000:1.0:1783520936.206198:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435038efb80/2, rc 0, transno 4294976417 00000100:00100000:1.0:1783520936.206214:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435038efb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993266432:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.206216:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.206238:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.206244:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0268cb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x865:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.206245:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002475776 00000100:00100000:0.0:1783520936.206250:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002475776 00010000:00010000:3.0:1783520936.206257:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.206261:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0268cb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x865:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.206271:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c39dae6d/0xd5401cbabc02202b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x864:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f273e expref: 160 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.206272:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0268cb lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x865:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.206275:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515476d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993268352:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.206301:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0268cb lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x865:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.206310:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0268cb lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x865:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.206310:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.206317:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36507 00000100:00100000:0.0:1783520936.206334:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503370380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+160:2388:x1870153002475776:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.206338:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.206342:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.206343:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.206345:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.206349:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c39dae6d/0xd5401cbabc02202b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x864:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f273e expref: 160 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.206402:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c39dae6d/0xd5401cbabc02202b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x864:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f273e expref: 160 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.206415:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.206420:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993268352 00010000:00010000:0.0:1783520936.206420:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c39dae6d/0xd5401cbabc02202b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x864:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f273e expref: 160 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.206426:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993268352 00010000:00010000:0.0:1783520936.206437:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x864:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.206441:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x864:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.206446:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0268d2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x864:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.206454:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0268d2 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x864:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.206455:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002475904 00000100:00100000:1.0:1783520936.206461:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44923 00000100:00100000:3.0:1783520936.206462:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002475904 00000100:00100000:1.0:1783520936.206473:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943506228a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+849:3626:x1870152993268352:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.206499:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2156 count 1 00010000:00010000:3.0:1783520936.206500:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000132e862b/0xd5401cbabc022a73 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x866:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3163 expref: 160 pid: 9151 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.206502:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.206516:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026903 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x86c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.206528:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0268d2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x864:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.206531:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002d66c7c2/0xd5401cbabc02207f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x86c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f27ae expref: 160 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.206535:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36508 00000100:00100000:0.0:1783520936.206539:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503370380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+160:2388:x1870153002475776:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 207us (434us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.206550:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515476680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+160:2388:x1870153002475904:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.206551:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0268d2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x864:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.206558:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x86c:0x0].0x0 since=38s v=(0 0) 00000100:00100000:0.0:1783520936.206561:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36507 00010000:00010000:3.0:1783520936.206565:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.206568:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x86c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.206574:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0268d2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x864:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.206575:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026903 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x86c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.206598:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502c9a680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993267456:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.206602:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0268d2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x864:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.206618:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002d66c7c2/0xd5401cbabc02207f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x86c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27ae expref: 159 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.206618:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0268e0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x867:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.206619:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.206643:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002d66c7c2/0xd5401cbabc02207f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x86c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27ae expref: 159 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.206653:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520936.206658:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530d500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993268480:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.206660:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0268d2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x864:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.206670:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbaa00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.206674:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0268d2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x864:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.206682:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0268d2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x864:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.206695:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0268d2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x864:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.206701:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470cc0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.206702:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0268d2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x864:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.206713:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436342b8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+850:3626:x1870152993266304:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3007us (3127us total) trans 4294976418 rc 0/0 00000100:00100000:2.0:1783520936.206722:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44915 00000004:00080000:1.0:1783520936.206726:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435038ece00/2, rc 0, transno 4294976418 00010000:00010000:0.0:1783520936.206732:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.206735:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.206738:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.206739:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435038ece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993266304:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.206795:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993268608:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.206803:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000132e862b/0xd5401cbabc022a73 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x866:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3163 expref: 159 pid: 9151 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.206818:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000132e862b/0xd5401cbabc022a73 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x866:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3163 expref: 159 pid: 9151 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.206836:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000132e862b/0xd5401cbabc022a73 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x866:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3163 expref: 159 pid: 9151 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.206836:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.206882:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515474380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993267584:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.206901:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0268e7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x868:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.206903:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d29880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993267712:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.206912:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x866:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.206920:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x866:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.206927:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0268d9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x866:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.206927:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0268ee lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x869:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.206943:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515476a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993268096:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.206949:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0268d9 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x866:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.206956:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993268608 00000100:00100000:1.0:1783520936.206960:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993268608 00010000:00010000:2.0:1783520936.206963:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0268f5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x86a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.206971:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.206989:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0268d9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x866:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.206990:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44924 00000100:00100000:1.0:1783520936.206999:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+850:3627:x1870152993268608:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.207005:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515476680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+158:2388:x1870153002475904:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 456us (588us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.207007:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0268d9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x866:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.207012:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2157 count 1 00010000:00010000:2.0:1783520936.207026:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0268d9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x866:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.207026:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc02690a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x86d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.207046:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003809b3d4/0xd5401cbabc022086 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x86d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f27bc expref: 158 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.207065:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x86d:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.207067:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0268d9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x866:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.207070:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x86d:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.207073:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36508 00010000:00010000:1.0:1783520936.207078:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc02690a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x86d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.207099:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.207104:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.207104:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003809b3d4/0xd5401cbabc022086 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x86d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27bc expref: 157 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.207112:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0268d9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x866:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.207116:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003809b3d4/0xd5401cbabc022086 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x86d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27bc expref: 157 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.207135:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520936.207140:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d29880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993268736:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.207141:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0268d9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x866:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.207153:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0268d9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x866:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.207163:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470a80 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.207167:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0268d9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x866:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.207177:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc0268d9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x866:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.207193:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94360550c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+851:3627:x1870152993266560:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3187us (3327us total) trans 4294976419 rc 0/0 00010000:00010000:3.0:1783520936.207194:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.207204:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44917 00000004:00080000:0.0:1783520936.207221:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94360550f100/2, rc 0, transno 4294976419 00000100:00100000:0.0:1783520936.207235:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94360550f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993266560:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.207247:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.207259:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.207265:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.207274:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002476032 00000100:00100000:0.0:1783520936.207279:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002476032 00000100:00100000:1.0:1783520936.207287:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1783520936.207297:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435161c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993268864:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.207300:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000065b19b2b/0xd5401cbabc02205c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x867:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2768 expref: 158 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.207311:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.207335:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.207337:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002476160 00000100:00100000:0.0:1783520936.207341:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002476160 00010000:00010000:0.0:1783520936.207353:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000033652089/0xd5401cbabc022063 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x868:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2776 expref: 159 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.207401:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36509 00000100:00100000:0.0:1783520936.207414:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515476a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+159:2389:x1870153002476032:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.207415:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36510 00010000:00010000:0.0:1783520936.207427:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.207429:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515474380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+159:2389:x1870153002476160:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.207436:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000065b19b2b/0xd5401cbabc02205c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x867:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2768 expref: 159 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.207442:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.207452:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000033652089/0xd5401cbabc022063 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x868:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2776 expref: 159 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.207454:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000065b19b2b/0xd5401cbabc02205c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x867:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2768 expref: 159 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.207476:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000033652089/0xd5401cbabc022063 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x868:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2776 expref: 159 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.207482:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000065b19b2b/0xd5401cbabc02205c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x867:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2768 expref: 159 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.207509:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000033652089/0xd5401cbabc022063 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x868:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2776 expref: 158 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.207511:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x867:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.207519:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x867:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.207527:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0268e0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x867:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.207532:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x868:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.207539:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x868:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.207539:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0268e0 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x867:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.207547:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0268e7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x868:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.207568:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0268e7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x868:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.207585:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.207596:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.207610:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0268e0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x867:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.207617:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515476a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+157:2389:x1870153002476032:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 205us (369us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.207618:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0268e7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x868:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.207626:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515474380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+157:2389:x1870153002476160:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 196us (315us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.207629:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0268e0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x867:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.207640:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36509 00010000:00010000:3.0:1783520936.207646:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0268e7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x868:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.207653:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36510 00010000:00010000:2.0:1783520936.207654:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0268e0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x867:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.207665:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0268e0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x867:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.207681:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993268864 00010000:00010000:3.0:1783520936.207688:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0268e7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x868:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.207688:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993268864 00010000:00010000:3.0:1783520936.207700:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0268e7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x868:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.207701:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.207719:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44925 00010000:00010000:3.0:1783520936.207720:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:1.0:1783520936.207729:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635738850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+851:3627:x1870152993268864:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.207731:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520936.207733:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.207738:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0268e0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x867:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.207740:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0268e7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x868:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.207746:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2158 count 1 00010000:00010000:3.0:1783520936.207757:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.207759:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.207762:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0268e0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x867:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.207764:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0268e7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x868:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.207765:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026911 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x86e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.207786:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515475500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+852:3625:x1870152993267072:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2885us (3393us total) trans 4294976420 rc 0/0 00010000:00010000:0.0:1783520936.207791:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0268e7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x868:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.207796:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000005818462/0xd5401cbabc022962 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x86e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3052 expref: 155 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.207798:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44918 00010000:00010000:1.0:1783520936.207815:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x86e:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.207819:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x86e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.207819:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0268e7 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x868:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.207829:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026911 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x86e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.207833:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515474e00/2, rc 0, transno 4294976420 00010000:00010000:0.0:1783520936.207834:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc0268e7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x868:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.207846:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515474e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993267072:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.207861:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.207862:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000005818462/0xd5401cbabc022962 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x86e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3052 expref: 155 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.207865:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.207869:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.207879:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000005818462/0xd5401cbabc022962 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x86e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3052 expref: 155 pid: 178496 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.207880:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0268e0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x867:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.207893:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1783520936.207905:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515474000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993269120:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.207910:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0268e0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x867:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.207910:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.207917:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a5180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993268992:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.207928:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f7dd65e/0xd5401cbabc0268e0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x867:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.207947:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.207947:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.207952:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.207955:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.207977:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470480 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.207981:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002476288 00000100:00100000:0.0:1783520936.207986:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002476288 00000100:00100000:2.0:1783520936.208001:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993269120 00010000:00010000:0.0:1783520936.208004:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000064bc176d/0xd5401cbabc02206a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x869:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2784 expref: 156 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.208007:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993269120 00000100:00100000:0.0:1783520936.208032:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36511 00000100:00100000:2.0:1783520936.208042:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44926 00000100:00100000:0.0:1783520936.208044:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f2050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+156:2389:x1870153002476288:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.208053:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515477b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+852:3626:x1870152993269120:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.208056:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.208063:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000064bc176d/0xd5401cbabc02206a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x869:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2784 expref: 156 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.208067:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2159 count 1 00010000:00010000:0.0:1783520936.208079:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000064bc176d/0xd5401cbabc02206a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x869:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2784 expref: 156 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.208083:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026918 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x86f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.208092:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000064bc176d/0xd5401cbabc02206a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x869:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2784 expref: 156 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.208098:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004134b6f2/0xd5401cbabc02208d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x86f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f27d8 expref: 156 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.208102:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x869:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.208109:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x869:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.208111:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x86f:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.208113:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0268ee lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x869:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.208116:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x86f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.208126:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026918 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x86f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.208126:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0268ee lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x869:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.208164:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.208166:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004134b6f2/0xd5401cbabc02208d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x86f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27d8 expref: 155 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.208177:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004134b6f2/0xd5401cbabc02208d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x86f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27d8 expref: 155 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.208187:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0268ee lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x869:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.208194:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f2050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+155:2389:x1870153002476288:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 151us (288us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.208198:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94360550d180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993269248:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.208202:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0268ee lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x869:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.208213:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36511 00010000:00010000:1.0:1783520936.208215:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0268ee lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x869:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.208221:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0268ee lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x869:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.208236:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1300 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.208238:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.208258:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.208263:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0268ee lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x869:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.208269:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.208273:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0268ee lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x869:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.208279:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0268ee lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x869:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.208290:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d28a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+854:3626:x1870152993267200:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3367us (3446us total) trans 4294976421 rc 0/0 00010000:00010000:0.0:1783520936.208291:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0268ee lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x869:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.208300:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515475880/2, rc 0, transno 4294976421 00000100:00100000:3.0:1783520936.208307:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44919 00010000:00010000:0.0:1783520936.208309:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc0268ee lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x869:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.208316:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d2bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+854:3627:x1870152993267328:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3163us (3241us total) trans 4294976422 rc 0/0 00000100:00100000:2.0:1783520936.208318:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515475880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993267200:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.208326:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44920 00010000:00010000:0.0:1783520936.208334:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.208337:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.208338:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520936.208390:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514d2a680/2, rc 0, transno 4294976422 00000100:00100000:1.0:1783520936.208399:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993267328:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.208416:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94360550c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993269376:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.208443:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.208445:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993269504:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.208453:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.208475:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002476416 00000100:00100000:1.0:1783520936.208480:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.208481:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002476416 00010000:00010000:0.0:1783520936.208512:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000004b91623/0xd5401cbabc022071 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x86a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2792 expref: 155 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.208536:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993269376 00000100:00100000:2.0:1783520936.208541:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993269376 00000100:00100000:0.0:1783520936.208551:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36512 00000100:00100000:0.0:1783520936.208562:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516313b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+155:2389:x1870153002476416:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.208570:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993269504 00010000:00010000:0.0:1783520936.208570:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.208575:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993269504 00010000:00010000:0.0:1783520936.208578:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000004b91623/0xd5401cbabc022071 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x86a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2792 expref: 155 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.208591:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000004b91623/0xd5401cbabc022071 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x86a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2792 expref: 155 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.208600:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44927 00010000:00010000:0.0:1783520936.208607:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000004b91623/0xd5401cbabc022071 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x86a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2792 expref: 155 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.208611:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+854:3626:x1870152993269376:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.208627:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x86a:0x0].0x0 since=0s v=(0 0) 00002000:00080000:2.0:1783520936.208629:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2160 count 1 00010000:00010000:0.0:1783520936.208631:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x86a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.208638:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0268f5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.208643:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02691f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x870:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.208647:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0268f5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x86a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.208651:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44928 00000100:00100000:3.0:1783520936.208665:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+854:3627:x1870152993269504:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.208671:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b518a940/0xd5401cbabc02209b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x870:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f27e6 expref: 154 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.208681:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x870:0x0].0x0 since=38s v=(0 0) 00002000:00080000:3.0:1783520936.208683:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2161 count 1 00010000:00010000:0.0:1783520936.208684:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.208685:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x870:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.208689:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0268f5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.208693:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02691f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x870:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.208701:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026926 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x871:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.208715:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516313b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+154:2389:x1870153002476416:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 153us (271us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.208730:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0268f5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.208735:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000061370da0/0xd5401cbabc0220a2 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x871:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f27f4 expref: 154 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.208737:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b518a940/0xd5401cbabc02209b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x870:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27e6 expref: 154 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.208737:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36512 00010000:00010000:1.0:1783520936.208757:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0268f5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.208762:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x871:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.208772:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x871:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.208782:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026926 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x871:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.208783:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.208787:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0268f5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x86a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.208792:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b518a940/0xd5401cbabc02209b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x870:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27e6 expref: 153 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.208819:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.208820:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9d500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993269632:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.208825:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.208832:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0268f5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x86a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.208840:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.208844:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0268f5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x86a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.208855:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0268f5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x86a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.208869:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0268f5 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x86a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.208878:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0268f5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x86a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.208878:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b19c0 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.208904:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.208906:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515474700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+855:3628:x1870152993267840:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3324us (3437us total) trans 4294976423 rc 0/0 00010000:00010000:3.0:1783520936.208909:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.208915:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.208916:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44921 00000004:00080000:2.0:1783520936.208925:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514d29c00/2, rc 0, transno 4294976423 00000100:00100000:0.0:1783520936.208938:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002476544 00000100:00100000:2.0:1783520936.208939:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993267840:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.208943:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000061370da0/0xd5401cbabc0220a2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x871:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27f4 expref: 153 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.208945:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002476544 00010000:00010000:3.0:1783520936.208957:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000061370da0/0xd5401cbabc0220a2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x871:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27f4 expref: 154 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.208965:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ebe32b4c/0xd5401cbabc022078 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x86b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27a0 expref: 154 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.208990:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36513 00000100:00100000:2.0:1783520936.208992:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993269888:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.209001:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516311c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+154:2390:x1870153002476544:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.209011:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.209015:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ebe32b4c/0xd5401cbabc022078 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x86b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27a0 expref: 154 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.209027:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.209031:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ebe32b4c/0xd5401cbabc022078 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x86b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27a0 expref: 154 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.209049:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503371500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993269760:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.209052:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993269888 00010000:00010000:0.0:1783520936.209054:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x86b:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.209058:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993269888 00010000:00010000:0.0:1783520936.209059:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x86b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.209066:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0268fc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.209075:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0268fc lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x86b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.209091:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.209092:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57d80 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.209094:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44929 00000100:00100000:2.0:1783520936.209101:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.209108:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+855:3628:x1870152993269888:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.209116:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516311c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+154:2390:x1870153002476544:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 117us (338us total) trans 0 rc 0/0 00002000:00080000:1.0:1783520936.209124:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2162 count 1 00000100:00100000:0.0:1783520936.209125:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36513 00010000:00010000:1.0:1783520936.209140:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02692d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x872:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.209145:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002476672 00000100:00100000:0.0:1783520936.209150:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002476672 00010000:00010000:1.0:1783520936.209156:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005c289dcd/0xd5401cbabc0220a9 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x872:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2802 expref: 154 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.209163:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x872:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.209167:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002d66c7c2/0xd5401cbabc02207f lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x86c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27ae expref: 154 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.209168:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x872:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.209177:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02692d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x872:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.209200:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36514 00010000:00010000:1.0:1783520936.209204:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005c289dcd/0xd5401cbabc0220a9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x872:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2802 expref: 154 pid: 35282 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.209213:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9d880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+154:2388:x1870153002476672:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.209220:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005c289dcd/0xd5401cbabc0220a9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x872:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2802 expref: 154 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.209223:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.209232:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002d66c7c2/0xd5401cbabc02207f lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x86c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27ae expref: 154 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.209239:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993270016:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.209245:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d29880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993268736:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.209247:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002d66c7c2/0xd5401cbabc02207f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x86c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27ae expref: 154 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.209264:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x86c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.209267:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x86c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.209269:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc02690a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x86d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.209272:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026903 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.209284:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470900 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.209292:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026903 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x86c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.209306:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.209325:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9d880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+154:2388:x1870153002476672:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 116us (230us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.209331:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a5180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993268992:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.209337:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36514 00010000:00010000:1.0:1783520936.209351:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026911 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x86e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.209397:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002d66c7c2/0xd5401cbabc02207f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x86c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27ae expref: 153 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.209409:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530d500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993268480:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.209424:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026903 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.209432:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026903 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.209440:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026903 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.209448:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026903 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.209456:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026903 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x86c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.209472:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.209486:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.209505:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026903 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x86c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.209512:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.209516:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026903 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x86c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.209525:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026903 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x86c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.209542:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026903 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x86c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.209551:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026903 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x86c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.209569:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943506228a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+856:3626:x1870152993268352:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3097us (3260us total) trans 4294976424 rc 0/0 00010000:00010000:1.0:1783520936.209574:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.209579:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.209579:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44923 00010000:00010000:1.0:1783520936.209585:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.209619:0:178496:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ebe32b4c/0xd5401cbabc022078 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x86b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27a0 expref: 152 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.209637:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516312a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993268224:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.209653:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0268fc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.209663:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0268fc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.209671:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0268fc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.209681:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0268fc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.209689:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0268fc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x86b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.209698:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515476d80/2, rc 0, transno 4294976424 00010000:00010000:0.0:1783520936.209706:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520936.209713:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515476d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993268352:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.209717:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.209724:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0268fc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x86b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.209733:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbebe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.209737:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0268fc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x86b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.209746:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0268fc lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x86b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.209759:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0268fc lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x86b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.209768:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0268fc lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x86b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.209782:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350622b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+856:3626:x1870152993267968:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4071us (4160us total) trans 4294976425 rc 0/0 00000100:00100000:0.0:1783520936.209791:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44922 00010000:00010000:1.0:1783520936.209794:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.209798:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.209801:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520936.209869:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514d29f80/2, rc 0, transno 4294976425 00000100:00100000:3.0:1783520936.209884:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993267968:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.209892:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1783520936.209900:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.209906:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993270144:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.209926:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002476800 00000100:00100000:0.0:1783520936.209931:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002476800 00000100:00100000:1.0:1783520936.209939:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.209946:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.209952:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003809b3d4/0xd5401cbabc022086 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x86d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27bc expref: 152 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.209973:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002476928 00000100:00100000:3.0:1783520936.209977:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993270272:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.209979:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002476928 00000100:00100000:0.0:1783520936.209985:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36515 00000100:00100000:0.0:1783520936.209995:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513bc0000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+153:2387:x1870153002476800:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.210005:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.210006:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000005818462/0xd5401cbabc022962 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x86e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3052 expref: 153 pid: 178496 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.210012:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.210014:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003809b3d4/0xd5401cbabc022086 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x86d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27bc expref: 153 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.210037:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003809b3d4/0xd5401cbabc022086 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x86d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27bc expref: 153 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.210039:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993270144 00000100:00100000:1.0:1783520936.210045:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993270144 00000100:00100000:2.0:1783520936.210051:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36516 00000100:00100000:3.0:1783520936.210056:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993270272 00010000:00010000:0.0:1783520936.210057:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003809b3d4/0xd5401cbabc022086 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x86d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27bc expref: 153 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.210063:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993270272 00000100:00100000:2.0:1783520936.210065:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a5180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+153:2387:x1870153002476928:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.210079:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.210079:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44930 00010000:00010000:0.0:1783520936.210080:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x86d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.210087:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x86d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.210090:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000005818462/0xd5401cbabc022962 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x86e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3052 expref: 152 pid: 178496 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.210093:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+857:3626:x1870152993270144:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.210095:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc02690a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.210103:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44931 00002000:00080000:1.0:1783520936.210116:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2163 count 1 00000100:00100000:3.0:1783520936.210117:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503370380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+857:3626:x1870152993270272:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.210117:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc02690a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x86d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.210120:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000005818462/0xd5401cbabc022962 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x86e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3052 expref: 152 pid: 178496 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.210133:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026934 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x873:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.210139:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2164 count 1 00010000:00010000:0.0:1783520936.210155:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.210159:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02693b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x874:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.210160:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000005818462/0xd5401cbabc022962 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x86e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3052 expref: 152 pid: 178496 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.210160:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ac3efeb8/0xd5401cbabc0220b0 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x873:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2810 expref: 152 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.210182:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513bc0000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+152:2387:x1870153002476800:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 186us (280us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.210188:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x873:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.210190:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001e2fa1f9/0xd5401cbabc0220b7 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x874:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f281e expref: 151 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.210192:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x86e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.210196:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x873:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.210200:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x86e:0x0].0x0 since=0s v=(1 0) 00000100:00100000:0.0:1783520936.210200:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36515 00010000:00010000:1.0:1783520936.210204:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026934 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x873:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.210211:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026911 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.210222:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc02690a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.210226:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x874:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.210235:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x874:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.210239:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026911 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x86e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.210241:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc02690a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.210246:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ac3efeb8/0xd5401cbabc0220b0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x873:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2810 expref: 150 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.210248:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02693b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x874:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.210271:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc02690a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.210277:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ac3efeb8/0xd5401cbabc0220b0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x873:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2810 expref: 150 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.210280:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.210287:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc02690a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x86d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.210299:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001e2fa1f9/0xd5401cbabc0220b7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x874:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f281e expref: 150 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.210310:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a5180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+150:2387:x1870153002476928:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 246us (369us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.210317:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.210320:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350d166a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993270400:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.210328:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001e2fa1f9/0xd5401cbabc0220b7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x874:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f281e expref: 150 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.210330:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36516 00010000:00010000:0.0:1783520936.210330:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.210337:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc02690a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x86d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.210346:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbbe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.210350:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc02690a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x86d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.210378:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc02690a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x86d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.210387:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470480 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.210392:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc02690a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x86d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.210406:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc02690a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x86d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.210420:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026911 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.210422:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.210425:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.210429:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.210437:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503373480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993270528:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.210437:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026911 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.210451:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026911 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.210457:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026911 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x86e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.210471:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520936.210476:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f180 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.210482:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.210500:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026911 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x86e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.210511:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.210516:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026911 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x86e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.210526:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026911 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x86e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.210534:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+859:3627:x1870152993268608:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3533us (3697us total) trans 4294976426 rc 0/0 00010000:00010000:2.0:1783520936.210543:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026911 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x86e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.210544:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44924 00010000:00010000:2.0:1783520936.210555:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026911 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x86e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.210576:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635738850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+858:3627:x1870152993268864:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2847us (3239us total) trans 4294976427 rc 0/0 00010000:00010000:2.0:1783520936.210577:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520936.210582:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435161c2a00/2, rc 0, transno 4294976427 00010000:00010000:2.0:1783520936.210583:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.210587:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.210589:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44925 00000100:00100000:0.0:1783520936.210605:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435161c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993268864:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.210619:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94360550d180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993269248:192.168.204.33@tcp:104: 00000004:00080000:0.0:1783520936.210628:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351530c380/2, rc 0, transno 4294976426 00000100:00100000:0.0:1783520936.210635:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993268608:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.210649:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026918 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x86f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.210709:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993270656:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.210722:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94360550c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993270784:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.210739:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9d500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993269632:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.210747:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.210757:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02691f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x870:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.210775:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.210777:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993270656 00000100:00100000:1.0:1783520936.210782:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993270656 00000100:00100000:1.0:1783520936.210809:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44932 00000100:00100000:1.0:1783520936.210816:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516312a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+858:3627:x1870152993270656:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.210829:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2165 count 1 00010000:00010000:1.0:1783520936.210842:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026942 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x875:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.210856:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a6786220/0xd5401cbabc0220be lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x875:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f282c expref: 149 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.210863:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x875:0x0].0x0 since=38s v=(0 0) 00000100:00100000:0.0:1783520936.210864:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993270784 00010000:00010000:1.0:1783520936.210868:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x875:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.210870:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993270784 00010000:00010000:1.0:1783520936.210874:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026942 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x875:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.210898:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a6786220/0xd5401cbabc0220be lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x875:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f282c expref: 149 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.210903:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44933 00010000:00010000:1.0:1783520936.210914:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a6786220/0xd5401cbabc0220be lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x875:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f282c expref: 149 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.210916:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+859:3628:x1870152993270784:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.210933:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ae5f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993270912:192.168.204.33@tcp:104: 00002000:00080000:0.0:1783520936.210934:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2166 count 1 00010000:00010000:0.0:1783520936.210949:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026949 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x876:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.210960:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000063375755/0xd5401cbabc022bbc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x876:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f32ac expref: 149 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.210972:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x876:0x0].0x0 since=37s v=(0 0) 00000100:00100000:1.0:1783520936.210975:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124706c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.210976:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x876:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.210983:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026949 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x876:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.211005:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000063375755/0xd5401cbabc022bbc lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x876:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f32ac expref: 149 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.211015:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000063375755/0xd5401cbabc022bbc lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x876:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f32ac expref: 149 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.211026:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516312680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993271040:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.211056:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80900 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.211065:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503371500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993269760:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.211143:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026926 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x871:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.211849:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.211889:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993270016:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.211907:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02692d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x872:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.211908:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002477056 00000100:00100000:2.0:1783520936.211915:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002477056 00000100:00100000:3.0:1783520936.211934:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.211940:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004134b6f2/0xd5401cbabc02208d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x86f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27d8 expref: 150 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.211964:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.212032:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002477184 00000100:00100000:0.0:1783520936.212032:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002477312 00000100:00100000:1.0:1783520936.212037:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002477184 00000100:00100000:0.0:1783520936.212039:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002477312 00000100:00100000:3.0:1783520936.212052:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.212059:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b518a940/0xd5401cbabc02209b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x870:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27e6 expref: 152 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.212064:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000061370da0/0xd5401cbabc0220a2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x871:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27f4 expref: 152 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.212104:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002477440 00000100:00100000:0.0:1783520936.212108:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002477440 00000100:00100000:1.0:1783520936.212115:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36517 00010000:00010000:0.0:1783520936.212125:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005c289dcd/0xd5401cbabc0220a9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x872:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2802 expref: 153 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.212129:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+153:2389:x1870153002477056:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520936.212139:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503373480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993270528:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.212142:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.212151:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004134b6f2/0xd5401cbabc02208d lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x86f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27d8 expref: 153 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.212160:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02693b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x874:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.212166:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004134b6f2/0xd5401cbabc02208d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x86f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27d8 expref: 153 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.212169:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36518 00000100:00100000:0.0:1783520936.212183:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503372a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+153:2388:x1870153002477312:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.212189:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004134b6f2/0xd5401cbabc02208d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x86f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27d8 expref: 153 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.212194:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.212202:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000061370da0/0xd5401cbabc0220a2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x871:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27f4 expref: 153 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.212222:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x86f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.212223:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000061370da0/0xd5401cbabc0220a2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x871:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27f4 expref: 152 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.212230:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x86f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.212241:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026918 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.212247:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000061370da0/0xd5401cbabc0220a2 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x871:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27f4 expref: 152 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.212255:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026918 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x86f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.212263:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x871:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.212268:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x871:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.212272:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026926 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x871:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.212280:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026926 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x871:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.212281:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.212293:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026918 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.212304:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026918 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.212304:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.212318:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+151:2389:x1870153002477056:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 192us (467us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.212319:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026918 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x86f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.212331:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503372a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+151:2388:x1870153002477312:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 148us (364us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.212336:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36517 00010000:00010000:3.0:1783520936.212340:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026918 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x86f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.212349:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36518 00010000:00010000:3.0:1783520936.212390:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.212467:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36519 00010000:00010000:1.0:1783520936.212478:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.212485:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026918 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x86f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.212506:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.212511:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026918 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x86f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.212523:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026918 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x86f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.212525:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36520 00000100:00100000:0.0:1783520936.212539:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503370e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+149:2390:x1870153002477440:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.212541:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026918 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x86f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.212549:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.212551:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026918 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x86f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.212558:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005c289dcd/0xd5401cbabc0220a9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x872:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2802 expref: 149 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.212571:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005c289dcd/0xd5401cbabc0220a9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x872:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2802 expref: 149 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.212583:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.212586:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.212588:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.212599:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005c289dcd/0xd5401cbabc0220a9 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x872:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2802 expref: 149 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.212601:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515477b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+860:3626:x1870152993269120:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4464us (4570us total) trans 4294976428 rc 0/0 00000100:00100000:1.0:1783520936.212610:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f2850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+149:2389:x1870153002477184:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520936.212621:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44926 00010000:00010000:0.0:1783520936.212622:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x872:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.212624:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.212628:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x872:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.212633:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b518a940/0xd5401cbabc02209b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x870:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27e6 expref: 148 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.212635:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02692d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x872:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.212655:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02692d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x872:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.212657:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b518a940/0xd5401cbabc02209b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x870:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27e6 expref: 148 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.212683:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b518a940/0xd5401cbabc02209b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x870:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f27e6 expref: 148 pid: 35282 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.212687:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.212699:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x870:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.212703:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x870:0x0].0x0 since=0s v=(0 0) 00000004:00080000:3.0:1783520936.212709:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515474000/2, rc 0, transno 4294976428 00010000:00010000:1.0:1783520936.212709:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02691f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x870:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.212723:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503370e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+147:2390:x1870153002477440:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 174us (659us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.212724:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02691f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x870:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.212728:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515474000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993269120:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.212740:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36520 00010000:00010000:1.0:1783520936.212759:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.212774:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026926 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x871:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.212784:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026926 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x871:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.212785:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f2850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+146:2389:x1870153002477184:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 309us (852us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.212807:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026926 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x871:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.212813:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36519 00000100:00100000:3.0:1783520936.212818:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.212823:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026926 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x871:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.212845:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.212858:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.212862:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026926 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x871:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.212868:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.212870:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026926 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x871:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.212876:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026926 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x871:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.212889:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026926 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x871:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.212894:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026926 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x871:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.212908:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.212910:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.212912:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.212921:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002477568 00000100:00100000:3.0:1783520936.212925:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.212925:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002477568 00000100:00100000:0.0:1783520936.212929:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+860:3627:x1870152993269504:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4266us (4448us total) trans 4294976429 rc 0/0 00000100:00100000:0.0:1783520936.212939:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44928 00010000:00010000:1.0:1783520936.212940:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ac3efeb8/0xd5401cbabc0220b0 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x873:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2810 expref: 146 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.212966:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002477696 00000100:00100000:1.0:1783520936.212969:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002477696 00000004:00080000:0.0:1783520936.212980:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351575dc00/2, rc 0, transno 4294976429 00010000:00010000:1.0:1783520936.212983:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001e2fa1f9/0xd5401cbabc0220b7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x874:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f281e expref: 147 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.212987:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993269504:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.213013:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36521 00000100:00100000:2.0:1783520936.213021:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36522 00000100:00100000:1.0:1783520936.213025:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503373480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+147:2388:x1870153002477568:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.213035:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.213040:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ac3efeb8/0xd5401cbabc0220b0 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x873:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2810 expref: 147 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.213059:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ac3efeb8/0xd5401cbabc0220b0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x873:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2810 expref: 147 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.213075:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993271296:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.213076:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x873:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.213085:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.213085:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x873:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.213092:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026934 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x873:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.213101:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.213102:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026934 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x873:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.213117:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.213141:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1783520936.213147:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94360550d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993271168:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.213157:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503373480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+147:2388:x1870153002477568:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 135us (344us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.213167:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.213168:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36521 00000100:00100000:1.0:1783520936.213186:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002477952 00000100:00100000:1.0:1783520936.213190:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002477952 00000100:00100000:2.0:1783520936.213198:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+147:2388:x1870153002477696:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.213206:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.213207:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000063375755/0xd5401cbabc022bbc lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x876:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f32ac expref: 147 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.213213:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001e2fa1f9/0xd5401cbabc0220b7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x874:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f281e expref: 147 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.213230:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001e2fa1f9/0xd5401cbabc0220b7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x874:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f281e expref: 147 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.213231:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.213249:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002477824 00000100:00100000:0.0:1783520936.213255:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002477824 00000100:00100000:1.0:1783520936.213256:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36523 00000100:00100000:1.0:1783520936.213266:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503372300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+148:2387:x1870153002477952:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.213274:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a6786220/0xd5401cbabc0220be lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x875:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f282c expref: 148 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.213275:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.213283:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000063375755/0xd5401cbabc022bbc lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x876:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f32ac expref: 148 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.213296:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000063375755/0xd5401cbabc022bbc lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x876:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f32ac expref: 148 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.213300:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993271296 00000100:00100000:0.0:1783520936.213302:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36524 00000100:00100000:3.0:1783520936.213308:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993271296 00000100:00100000:0.0:1783520936.213315:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+148:2387:x1870153002477824:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.213316:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x876:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.213324:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x876:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.213328:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.213329:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026949 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x876:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.213337:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993271168 00010000:00010000:0.0:1783520936.213337:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a6786220/0xd5401cbabc0220be lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x875:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f282c expref: 148 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.213342:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993271168 00010000:00010000:1.0:1783520936.213342:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026949 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x876:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.213399:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44934 00010000:00010000:0.0:1783520936.213410:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a6786220/0xd5401cbabc0220be lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x875:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f282c expref: 148 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.213411:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+861:3626:x1870152993271296:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.213430:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x875:0x0].0x0 since=0s v=(0 0) 00002000:00080000:3.0:1783520936.213433:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2168 count 1 00010000:00010000:0.0:1783520936.213434:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x875:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.213442:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026942 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x875:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.213450:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026942 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x875:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.213461:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.213467:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.213503:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503372300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+148:2387:x1870153002477952:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 237us (422us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.213508:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+148:2387:x1870153002477824:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 193us (277us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.213516:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36523 00010000:00010000:3.0:1783520936.213523:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026950 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x878:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.213538:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ce6fdb20/0xd5401cbabc0220d3 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x878:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2856 expref: 147 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.213552:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x878:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.213554:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02692d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x872:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.213556:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x878:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.213566:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026950 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x878:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.213569:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02692d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x872:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.213594:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02692d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x872:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.213601:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001e2fa1f9/0xd5401cbabc0220b7 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x874:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f281e expref: 147 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.213604:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02692d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x872:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.213609:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ce6fdb20/0xd5401cbabc0220d3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x878:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2856 expref: 147 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.213625:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x874:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.213633:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x874:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.213637:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02693b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x874:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.213641:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ce6fdb20/0xd5401cbabc0220d3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x878:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2856 expref: 146 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.213641:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.213646:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02693b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x874:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.213670:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435038ec380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993271424:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.213676:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.213701:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+146:2388:x1870153002477696:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 672us (780us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.213719:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f180 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.213733:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+862:3628:x1870152993269888:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4627us (4708us total) trans 4294976430 rc 0/0 00000100:00100000:0.0:1783520936.213735:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36524 00000100:00100000:1.0:1783520936.213742:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44929 00000004:00080000:3.0:1783520936.213749:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e9ca80/2, rc 0, transno 4294976430 00000100:00100000:1.0:1783520936.213760:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44935 00010000:00010000:0.0:1783520936.213761:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520936.213766:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514d28a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+861:3626:x1870152993271168:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.213768:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02692d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x872:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.213769:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993269888:0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.213777:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2167 count 1 00010000:00010000:0.0:1783520936.213785:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.213789:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02692d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x872:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.213792:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026957 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x877:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.213808:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02692d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x872:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.213815:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000fd22d670/0xd5401cbabc0220cc lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x877:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2848 expref: 145 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.213827:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x877:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.213828:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02692d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x872:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.213832:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x877:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.213841:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026957 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x877:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.213858:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc02692d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x872:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.213866:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02693b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x874:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.213874:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000fd22d670/0xd5401cbabc0220cc lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x877:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2848 expref: 145 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.213876:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.213880:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02693b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x874:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.213881:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.213884:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.213897:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000fd22d670/0xd5401cbabc0220cc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x877:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2848 expref: 145 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.213898:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02693b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x874:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.213915:0:35275:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000063375755/0xd5401cbabc022bbc lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x876:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 62914560->62918655) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f32ac expref: 145 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.213938:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ae4380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993271552:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.213940:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516312680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993271040:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.213965:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02693b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x874:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.213965:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026949 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x876:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.213976:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.213991:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026949 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x876:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.213997:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.214000:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026949 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x876:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.214010:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026949 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x876:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.214017:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026949 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x876:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.214046:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520936.214053:0:8408:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ac3efeb8/0xd5401cbabc0220b0 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x873:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2810 expref: 144 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.214062:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350d166a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993270400:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.214073:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026934 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x873:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.214081:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1783520936.214084:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02693b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x874:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.214089:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.214091:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02693b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x874:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.214096:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02693b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x874:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.214105:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02693b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x874:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.214109:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc02693b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x874:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.214122:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.214125:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026949 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x876:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.214130:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.214132:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026949 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x876:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.214140:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026949 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x876:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.214142:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503370380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+863:3626:x1870152993270272:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4023us (4128us total) trans 4294976431 rc 0/0 00010000:00010000:1.0:1783520936.214149:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026949 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x876:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.214157:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026949 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x876:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.214158:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44931 00000100:00100000:0.0:1783520936.214158:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+863:3628:x1870152993270784:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3242us (3381us total) trans 4294976432 rc 0/0 00000100:00100000:0.0:1783520936.214175:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44933 00010000:00010000:1.0:1783520936.214186:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.214189:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.214192:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.214199:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026934 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x873:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.214201:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94360550c700/2, rc 0, transno 4294976432 00010000:00010000:1.0:1783520936.214208:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026934 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x873:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.214212:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94360550c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993270784:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.214225:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026934 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x873:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.214230:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026934 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x873:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.214237:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36522 00010000:00010000:1.0:1783520936.214246:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520936.214298:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993271808:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.214300:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.214309:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026934 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x873:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.214309:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+862:3626:x1870152993270144:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4220us (4363us total) trans 4294976433 rc 0/0 00000100:00100000:1.0:1783520936.214318:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44930 00010000:00010000:2.0:1783520936.214325:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.214330:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026934 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x873:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.214332:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.214343:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026934 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x873:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.214351:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a6786220/0xd5401cbabc0220be lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x875:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f282c expref: 142 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.214377:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:2.0:1783520936.214381:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026934 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x873:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.214387:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993271680:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.214396:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026934 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x873:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.214415:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.214418:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.214419:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.214423:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:2.0:1783520936.214447:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e9f100/2, rc 0, transno 4294976431 00000100:00100000:1.0:1783520936.214454:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ae5f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993270912:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.214463:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993270272:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.214470:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026942 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x875:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.214477:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e9ce00/2, rc 0, transno 4294976433 00010000:00010000:1.0:1783520936.214481:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026942 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x875:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.214484:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993270144:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.214507:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:1.0:1783520936.214509:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026942 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x875:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.214518:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350c5bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993271936:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.214520:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026942 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x875:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.214530:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026942 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x875:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.214544:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.214551:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.214595:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02691f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x870:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.214605:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02691f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x870:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.214614:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02691f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x870:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.214624:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02691f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x870:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.214625:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516312a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+862:3627:x1870152993270656:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3807us (3879us total) trans 4294976434 rc 0/0 00000100:00100000:1.0:1783520936.214639:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44932 00000100:00100000:2.0:1783520936.214654:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516681f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993272064:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.214656:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993271808 00000100:00100000:1.0:1783520936.214662:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993271808 00000100:00100000:2.0:1783520936.214687:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.214690:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993271680 00000100:00100000:1.0:1783520936.214695:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993271680 00000100:00100000:3.0:1783520936.214704:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993271936 00000100:00100000:3.0:1783520936.214709:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993271936 00000100:00100000:1.0:1783520936.214720:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993272064 00000100:00100000:1.0:1783520936.214724:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993272064 00000100:00100000:3.0:1783520936.214736:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44936 00010000:00010000:2.0:1783520936.214741:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:3.0:1783520936.214748:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435038ed180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+865:3628:x1870152993271808:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.214750:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1783520936.214757:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026942 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x875:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.214765:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.214768:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026942 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x875:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.214776:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026942 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x875:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.214788:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026942 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x875:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.214801:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026942 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x875:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.214813:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2170 count 1 00010000:00010000:1.0:1783520936.214826:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.214830:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02695e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x87a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.214830:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+866:3626:x1870152993269376:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6220us (6377us total) trans 4294976435 rc 0/0 00010000:00010000:1.0:1783520936.214832:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02691f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x870:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.214850:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44927 00010000:00010000:1.0:1783520936.214858:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.214861:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008ab3fcbc/0xd5401cbabc02215f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x87a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2872 expref: 141 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.214864:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02691f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x870:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.214886:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02691f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x870:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.214890:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94360550c000/2, rc 0, transno 4294976435 00010000:00010000:3.0:1783520936.214891:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x87a:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.214900:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x87a:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.214907:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94360550c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993269376:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.214907:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02691f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x870:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.214908:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02695e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x87a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.214929:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc02691f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x870:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.214946:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.214947:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.214948:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.215000:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008ab3fcbc/0xd5401cbabc02215f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x87a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2872 expref: 141 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.215016:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008ab3fcbc/0xd5401cbabc02215f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x87a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2872 expref: 141 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.215029:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993272192:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.215034:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44937 00010000:00010000:0.0:1783520936.215041:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.215050:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1783520936.215050:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435038eea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+865:3628:x1870152993271680:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.215053:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.215057:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.215065:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363fe54a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993272320:0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.215069:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2169 count 1 00010000:00010000:2.0:1783520936.215085:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026965 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x879:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.215089:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002478208 00000100:00100000:1.0:1783520936.215096:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002478208 00010000:00010000:2.0:1783520936.215101:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000079984e4b/0xd5401cbabc022158 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x879:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2864 expref: 142 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.215110:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x879:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.215114:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x879:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.215119:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026965 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x879:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.215120:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000fd22d670/0xd5401cbabc0220cc lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x877:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2848 expref: 142 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.215125:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:0.0:1783520936.215125:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e9e680/2, rc 0, transno 4294976434 00000100:00100000:0.0:1783520936.215164:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993270656:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.215173:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36525 00000100:00100000:0.0:1783520936.215187:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1783520936.215192:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f1050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+142:2387:x1870153002478208:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.215206:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.215206:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363fe54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993272448:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.215214:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000079984e4b/0xd5401cbabc022158 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x879:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2864 expref: 142 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.215219:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000fd22d670/0xd5401cbabc0220cc lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x877:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2848 expref: 142 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.215239:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000079984e4b/0xd5401cbabc022158 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x879:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2864 expref: 142 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.215241:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000fd22d670/0xd5401cbabc0220cc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x877:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2848 expref: 142 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.215267:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x877:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.215268:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363fe55c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993272576:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.215271:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x877:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.215278:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993272320 00010000:00010000:1.0:1783520936.215279:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026957 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x877:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.215286:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993272320 00000100:00100000:0.0:1783520936.215290:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.215299:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026957 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x877:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.215314:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.215338:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1300 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.215341:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f1050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+142:2387:x1870153002478208:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 157us (293us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.215351:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36525 00000100:00100000:3.0:1783520936.215402:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.215468:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993272448 00000100:00100000:3.0:1783520936.215472:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993272448 00000100:00100000:0.0:1783520936.215522:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44938 00000100:00100000:1.0:1783520936.215523:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002478080 00000100:00100000:1.0:1783520936.215530:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002478080 00000100:00100000:3.0:1783520936.215533:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44939 00000100:00100000:0.0:1783520936.215542:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517247100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+867:3626:x1870152993271936:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.215543:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44940 00000100:00100000:3.0:1783520936.215545:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517244e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+867:3626:x1870152993272064:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.215555:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ce6fdb20/0xd5401cbabc0220d3 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x878:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2856 expref: 142 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.215562:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350794ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+867:3628:x1870152993272320:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.215585:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2171 count 1 00002000:00080000:2.0:1783520936.215587:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2173 count 1 00000100:00100000:1.0:1783520936.215595:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36526 00000100:00100000:1.0:1783520936.215607:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350794df80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+142:2387:x1870153002478080:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.215613:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02696c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x87d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.215615:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2172 count 1 00010000:00010000:0.0:1783520936.215618:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026973 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x87b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.215620:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.215638:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ce6fdb20/0xd5401cbabc0220d3 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x878:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2856 expref: 142 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.215639:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02697a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x87c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.215670:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000010ad4f7b/0xd5401cbabc0221ba lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x87d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f289c expref: 142 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.215684:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004cc4a724/0xd5401cbabc0221ac lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x87b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2880 expref: 142 pid: 178496 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.215686:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f5c4c3ff/0xd5401cbabc0221b3 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x87c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f288e expref: 142 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.215691:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ce6fdb20/0xd5401cbabc0220d3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x878:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2856 expref: 142 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.215702:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x87d:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.215720:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x87d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.215741:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x87c:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.215741:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x87b:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.215742:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02696c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x87d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.215749:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x878:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.215763:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x87b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.215764:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x87c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.215764:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x878:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.215780:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000010ad4f7b/0xd5401cbabc0221ba lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x87d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f289c expref: 142 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.215783:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026973 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x87b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.215787:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02697a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x87c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.215787:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026950 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x878:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.215803:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000010ad4f7b/0xd5401cbabc0221ba lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x87d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f289c expref: 142 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.215839:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026950 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x878:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.215857:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f5c4c3ff/0xd5401cbabc0221b3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x87c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f288e expref: 142 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.215863:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e471f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993272704:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.215896:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f5c4c3ff/0xd5401cbabc0221b3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x87c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f288e expref: 142 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.215902:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.215913:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502250e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993272832:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.215919:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004cc4a724/0xd5401cbabc0221ac lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x87b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2880 expref: 142 pid: 178496 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.215929:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1180 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.215935:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350794df80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+142:2387:x1870153002478080:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 327us (573us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.215941:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004cc4a724/0xd5401cbabc0221ac lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x87b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2880 expref: 142 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.215950:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943505a57d80 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.215955:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44941 00000100:00100000:1.0:1783520936.215960:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36526 00000100:00100000:2.0:1783520936.215973:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516312680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+867:3627:x1870152993272448:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.215986:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff94351db63180 going to sleep for 11 seconds 00002000:00080000:2.0:1783520936.216000:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2174 count 1 00000100:00100000:0.0:1783520936.216002:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516310700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993272960:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.216012:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026981 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x87e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.216024:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000313d5838/0xd5401cbabc022aab lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x87e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f319b expref: 141 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.216026:0:20415:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ce6fdb20/0xd5401cbabc0220d3 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x878:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2856 expref: 141 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.216036:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x87e:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.216043:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x87e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.216047:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026981 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x87e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.216051:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435038ec380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993271424:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.216061:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80600 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.216072:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000313d5838/0xd5401cbabc022aab lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x87e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f319b expref: 140 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.216077:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026950 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x878:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.216083:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000313d5838/0xd5401cbabc022aab lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x87e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f319b expref: 140 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.216104:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e472300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993273088:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.216150:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1780 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.216151:0:35274:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000fd22d670/0xd5401cbabc0220cc lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x877:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2848 expref: 140 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.216168:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ae4380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993271552:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.216186:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026957 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x877:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.216187:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026950 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x878:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.216210:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026957 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x877:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.216220:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026957 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x877:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.216230:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026957 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x877:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.216238:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026957 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x877:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.216262:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.216269:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363fe55c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993272576:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.216279:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026965 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x879:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.216288:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.216291:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026957 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x877:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.216297:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.216299:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026957 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x877:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.216308:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026957 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x877:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.216318:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026957 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x877:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.216323:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026957 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x877:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.216336:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.216338:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.216339:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:2.0:1783520936.216387:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94360550d180/2, rc 0, transno 4294976436 00000100:00100000:1.0:1783520936.216387:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514d28a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+868:3626:x1870152993271168:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2594us (3194us total) trans 4294976436 rc 0/0 00000100:00100000:1.0:1783520936.216397:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44935 00000100:00100000:2.0:1783520936.216405:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94360550d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993271168:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.216465:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026950 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x878:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.216468:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:3.0:1783520936.216481:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026950 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x878:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.216485:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363fe55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993273216:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.216522:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.216547:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993273216 00000100:00100000:0.0:1783520936.216554:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993273216 00010000:00010000:3.0:1783520936.216576:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026950 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x878:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.216598:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.216610:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44942 00010000:00010000:2.0:1783520936.216613:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.216620:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026950 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x878:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.216626:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514fdc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+868:3626:x1870152993273216:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.216640:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.216644:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026950 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x878:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.216654:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2175 count 1 00010000:00010000:2.0:1783520936.216655:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026950 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x878:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.216673:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026950 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x878:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.216680:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026950 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x878:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.216680:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026988 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x87f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.216693:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+869:3626:x1870152993271296:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3279us (3585us total) trans 4294976437 rc 0/0 00000100:00100000:3.0:1783520936.216737:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44934 00010000:00010000:0.0:1783520936.216752:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c41ee974/0xd5401cbabc0221c8 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x87f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f28b8 expref: 139 pid: 178496 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.216755:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.216759:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.216764:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.216774:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x87f:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.216779:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x87f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.216784:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026988 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x87f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.216791:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e9d880/2, rc 0, transno 4294976437 00000100:00100000:2.0:1783520936.216805:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993271296:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.216816:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c41ee974/0xd5401cbabc0221c8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x87f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28b8 expref: 139 pid: 178496 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.216817:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.216841:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c41ee974/0xd5401cbabc0221c8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x87f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28b8 expref: 139 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.216868:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002478336 00000100:00100000:1.0:1783520936.216875:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002478336 00000100:00100000:0.0:1783520936.216875:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516313b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993273344:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.216884:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1783520936.216899:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514fdd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993273472:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.216909:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000079984e4b/0xd5401cbabc022158 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x879:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2864 expref: 140 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.216941:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.216964:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36527 00000100:00100000:0.0:1783520936.216973:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80b40 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.216980:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435038ec380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+140:2388:x1870153002478336:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.216993:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.216999:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000079984e4b/0xd5401cbabc022158 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x879:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2864 expref: 140 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.217002:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993273472 00000100:00100000:2.0:1783520936.217007:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993273472 00010000:00010000:1.0:1783520936.217022:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000079984e4b/0xd5401cbabc022158 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x879:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2864 expref: 140 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.217038:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44943 00010000:00010000:1.0:1783520936.217040:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000079984e4b/0xd5401cbabc022158 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x879:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2864 expref: 140 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.217048:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507938a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+869:3625:x1870152993273472:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.217062:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x879:0x0].0x0 since=0s v=(0 0) 00002000:00080000:2.0:1783520936.217063:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2176 count 1 00010000:00010000:1.0:1783520936.217069:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x879:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.217075:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026965 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x879:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.217077:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02698f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x880:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.217091:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026965 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x879:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.217105:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000901dfe02/0xd5401cbabc0221cf lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x880:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f28c6 expref: 139 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.217110:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x880:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.217113:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x880:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.217116:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02698f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x880:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.217119:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.217132:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000901dfe02/0xd5401cbabc0221cf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x880:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28c6 expref: 139 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.217141:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000901dfe02/0xd5401cbabc0221cf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x880:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28c6 expref: 139 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.217152:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350793bb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993273600:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.217158:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435038ec380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+139:2388:x1870153002478336:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 183us (348us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.217167:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36527 00000100:00100000:2.0:1783520936.217180:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1300 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.217199:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026965 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x879:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.217204:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026965 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x879:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.217209:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026965 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x879:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.217218:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026965 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x879:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.217234:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.217247:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.217254:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026965 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x879:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.217263:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.217268:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026965 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x879:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.217278:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026965 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x879:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.217292:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026965 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x879:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.217302:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026965 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x879:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.217311:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435038eea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+870:3628:x1870152993271680:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2265us (2891us total) trans 4294976438 rc 0/0 00000100:00100000:2.0:1783520936.217320:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44937 00010000:00010000:1.0:1783520936.217329:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.217336:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.217339:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.217401:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e471f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993272704:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.217428:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02696c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x87d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.217480:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514d2bb80/2, rc 0, transno 4294976438 00000100:00100000:3.0:1783520936.217509:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993271680:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.217546:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516310700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993272960:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.217555:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f2a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993272192:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.217581:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026973 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x87b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.217587:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02695e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x87a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.217589:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507939180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993273728:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.217610:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e472300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993273088:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.217633:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026981 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x87e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.217646:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502250e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993272832:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.217656:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.217674:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02697a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x87c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.217709:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993273728 00000100:00100000:1.0:1783520936.217714:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993273728 00000100:00100000:1.0:1783520936.217749:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44944 00000100:00100000:1.0:1783520936.217758:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516311c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+870:3627:x1870152993273728:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.217772:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2177 count 1 00010000:00010000:1.0:1783520936.217786:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026996 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x881:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.217799:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001c1885e3/0xd5401cbabc0221d6 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x881:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f28d4 expref: 138 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.217807:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x881:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.217810:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x881:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.217814:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026996 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x881:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.217834:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001c1885e3/0xd5401cbabc0221d6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x881:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28d4 expref: 138 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.217845:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001c1885e3/0xd5401cbabc0221d6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x881:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28d4 expref: 138 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.217857:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ae4380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993273856:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.217913:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470f00 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.217959:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516313b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993273344:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.217984:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026988 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x87f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.218174:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.218206:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002478464 00000100:00100000:2.0:1783520936.218209:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002478464 00010000:00010000:2.0:1783520936.218223:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000010ad4f7b/0xd5401cbabc0221ba lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x87d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f289c expref: 139 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.218244:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36528 00000100:00100000:2.0:1783520936.218248:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502250e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+139:2387:x1870153002478464:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.218254:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.218257:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000010ad4f7b/0xd5401cbabc0221ba lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x87d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f289c expref: 139 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.218264:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000010ad4f7b/0xd5401cbabc0221ba lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x87d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f289c expref: 139 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.218273:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000010ad4f7b/0xd5401cbabc0221ba lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x87d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f289c expref: 139 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.218281:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x87d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.218283:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x87d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.218285:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02696c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x87d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.218290:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02696c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x87d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.218306:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.218324:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502250e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+138:2387:x1870153002478464:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 76us (152us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.218328:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36528 00000100:00100000:3.0:1783520936.218415:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.218438:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002478720 00000100:00100000:1.0:1783520936.218444:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002478720 00010000:00010000:1.0:1783520936.218464:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008ab3fcbc/0xd5401cbabc02215f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x87a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2872 expref: 138 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.218482:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.218507:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002478976 00000100:00100000:1.0:1783520936.218512:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002478976 00000100:00100000:2.0:1783520936.218532:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.218532:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000313d5838/0xd5401cbabc022aab lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x87e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f319b expref: 139 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.218581:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.218602:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002478592 00000100:00100000:0.0:1783520936.218610:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002478592 00000100:00100000:1.0:1783520936.218611:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002478848 00000100:00100000:1.0:1783520936.218619:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002478848 00000100:00100000:2.0:1783520936.218634:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.218646:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004cc4a724/0xd5401cbabc0221ac lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x87b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2880 expref: 141 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.218656:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f5c4c3ff/0xd5401cbabc0221b3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x87c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f288e expref: 141 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.218681:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350793bb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993273600:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.218681:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002479232 00000100:00100000:2.0:1783520936.218685:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.218693:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002479232 00010000:00010000:3.0:1783520936.218717:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02698f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x880:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.218720:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000901dfe02/0xd5401cbabc0221cf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x880:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28c6 expref: 142 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.218725:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002479104 00000100:00100000:0.0:1783520936.218733:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002479104 00010000:00010000:0.0:1783520936.218758:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c41ee974/0xd5401cbabc0221c8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x87f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28b8 expref: 143 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.218771:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36529 00000100:00100000:2.0:1783520936.218788:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36530 00000100:00100000:1.0:1783520936.218788:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435079f2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+143:2387:x1870153002478720:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.218802:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.218807:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350793aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+143:2387:x1870153002478976:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.218811:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008ab3fcbc/0xd5401cbabc02215f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x87a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2872 expref: 143 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.218818:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36531 00010000:00010000:2.0:1783520936.218829:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.218844:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000313d5838/0xd5401cbabc022aab lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x87e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f319b expref: 143 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.218849:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943603d94850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+143:2387:x1870153002478848:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.218850:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008ab3fcbc/0xd5401cbabc02215f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x87a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2872 expref: 143 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.218879:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.218886:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000313d5838/0xd5401cbabc022aab lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x87e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f319b expref: 143 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.218890:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008ab3fcbc/0xd5401cbabc02215f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x87a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2872 expref: 143 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.218906:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004cc4a724/0xd5401cbabc0221ac lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x87b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2880 expref: 143 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.218924:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x87a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.218944:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x87a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.218946:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000313d5838/0xd5401cbabc022aab lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x87e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 41943040->41947135) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f319b expref: 142 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.218960:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02695e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x87a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.218980:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02695e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x87a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.218984:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x87e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.218989:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x87e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.218994:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026981 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x87e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.218995:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.219005:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026981 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x87e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.219023:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435079f2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+141:2387:x1870153002478720:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 237us (820us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.219029:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.219032:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36529 00010000:00010000:3.0:1783520936.219050:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02695e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x87a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.219056:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350793aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+140:2387:x1870153002478976:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 255us (573us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.219059:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36532 00000100:00100000:1.0:1783520936.219072:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507938380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+140:2387:x1870153002478592:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.219073:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02695e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x87a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.219077:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36530 00010000:00010000:3.0:1783520936.219090:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02695e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x87a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.219090:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.219101:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f5c4c3ff/0xd5401cbabc0221b3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x87c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f288e expref: 139 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.219102:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36533 00010000:00010000:3.0:1783520936.219104:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02695e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x87a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.219116:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507939c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+139:2387:x1870153002479232:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.219128:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f5c4c3ff/0xd5401cbabc0221b3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x87c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f288e expref: 139 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.219133:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.219140:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.219142:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000901dfe02/0xd5401cbabc0221cf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x880:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28c6 expref: 139 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.219142:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f5c4c3ff/0xd5401cbabc0221b3 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x87c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f288e expref: 139 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.219171:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000901dfe02/0xd5401cbabc0221cf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x880:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28c6 expref: 138 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.219171:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x87c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.219179:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x87c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.219189:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02697a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x87c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.219198:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000901dfe02/0xd5401cbabc0221cf lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x880:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28c6 expref: 138 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.219200:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02697a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x87c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.219221:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x880:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.219226:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x880:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.219233:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435038ed180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+871:3628:x1870152993271808:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4486us (4900us total) trans 4294976439 rc 0/0 00010000:00010000:1.0:1783520936.219233:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.219234:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02698f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x880:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.219249:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44936 00010000:00010000:0.0:1783520936.219251:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004cc4a724/0xd5401cbabc0221ac lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x87b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2880 expref: 137 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.219252:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02698f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x880:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.219259:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507938380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+137:2387:x1870153002478592:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 189us (729us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.219274:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02697a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x87c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.219274:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36532 00010000:00010000:0.0:1783520936.219281:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004cc4a724/0xd5401cbabc0221ac lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x87b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2880 expref: 137 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.219288:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.219293:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02697a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x87c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.219296:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36534 00010000:00010000:0.0:1783520936.219302:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x87b:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.219308:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350793ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+135:2387:x1870153002479104:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.219308:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x87b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.219309:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02697a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x87c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.219315:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026973 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x87b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.219316:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507939c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+135:2387:x1870153002479232:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 201us (679us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.219319:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.219327:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c41ee974/0xd5401cbabc0221c8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x87f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28b8 expref: 135 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.219332:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02697a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x87c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.219337:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026973 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x87b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.219351:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36533 00010000:00010000:1.0:1783520936.219355:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c41ee974/0xd5401cbabc0221c8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x87f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28b8 expref: 135 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.219394:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:0.0:1783520936.219396:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520936.219403:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02695e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x87a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.219411:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.219415:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02695e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x87a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.219424:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02695e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x87a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.219434:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.219434:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c41ee974/0xd5401cbabc0221c8 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x87f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28b8 expref: 134 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.219436:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02695e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x87a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.219442:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02697a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x87c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.219467:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.219468:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc02695e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x87a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.219470:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x87f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.219472:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02697a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x87c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.219474:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517244e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+871:3626:x1870152993272064:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3931us (4786us total) trans 4294976440 rc 0/0 00010000:00010000:1.0:1783520936.219477:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x87f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.219486:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026988 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x87f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.219508:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02697a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x87c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.219511:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026988 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x87f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.219513:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.219515:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.219518:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.219532:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02697a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x87c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.219537:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.219543:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc02697a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x87c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.219547:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.219552:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44939 00000100:00100000:1.0:1783520936.219562:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350793ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+133:2387:x1870153002479104:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 253us (893us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.219569:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.219572:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36534 00010000:00010000:2.0:1783520936.219574:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.219575:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943603d94850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+133:2387:x1870153002478848:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 742us (992us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.219578:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.219585:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36531 00000004:00080000:3.0:1783520936.219594:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516681f80/2, rc 0, transno 4294976440 00000004:00080000:2.0:1783520936.219609:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514d29880/2, rc 0, transno 4294976439 00000100:00100000:3.0:1783520936.219611:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516681f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993272064:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.219619:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993271808:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.219629:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ae4380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993273856:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.219645:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026996 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x881:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.219650:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026981 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x87e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.219671:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.219672:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026981 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x87e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.219684:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026981 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x87e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.219693:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026981 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x87e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.219700:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1783520936.219700:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1783520936.219716:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435079f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993273984:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.219717:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b04e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993274112:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.219731:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.219741:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.219746:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026981 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x87e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.219754:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.219759:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026981 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x87e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.219760:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.219765:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02698f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x880:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.219776:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026981 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x87e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.219781:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02698f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x880:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.219796:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02698f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x880:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.219796:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026981 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x87e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.219806:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026973 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x87b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.219818:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02698f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x880:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.219820:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026981 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x87e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.219827:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026973 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x87b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.219847:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026973 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x87b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.219851:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.219854:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.219858:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.219859:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026973 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x87b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.219884:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520936.219892:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.219893:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.219899:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026973 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x87b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.219909:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.219913:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026973 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x87b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.219926:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026973 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x87b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.219929:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993273984 00000100:00100000:3.0:1783520936.219936:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993273984 00010000:00010000:1.0:1783520936.219942:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026973 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x87b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.219951:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026973 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x87b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.219958:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516312680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+873:3627:x1870152993272448:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3988us (4678us total) trans 4294976441 rc 0/0 00000100:00100000:3.0:1783520936.219968:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993274112 00000100:00100000:2.0:1783520936.219969:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44941 00000004:00080000:0.0:1783520936.219969:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363fe54000/2, rc 0, transno 4294976441 00010000:00010000:1.0:1783520936.219971:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.219973:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993274112 00010000:00010000:1.0:1783520936.219976:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.219980:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.219991:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363fe54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993272448:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.219992:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002479360 00000100:00100000:1.0:1783520936.219996:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002479360 00000100:00100000:2.0:1783520936.220000:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44945 00000100:00100000:2.0:1783520936.220011:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516311880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+873:3627:x1870152993273984:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.220017:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001c1885e3/0xd5401cbabc0221d6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x881:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28d4 expref: 132 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.220030:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2178 count 1 00000100:00100000:0.0:1783520936.220033:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517247100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+873:3626:x1870152993271936:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4500us (5485us total) trans 4294976442 rc 0/0 00000100:00100000:3.0:1783520936.220042:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993274240:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.220044:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36535 00000100:00100000:0.0:1783520936.220044:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44938 00000100:00100000:1.0:1783520936.220055:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516310000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+132:2387:x1870153002479360:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.220064:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.220065:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44946 00010000:00010000:1.0:1783520936.220070:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001c1885e3/0xd5401cbabc0221d6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x881:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28d4 expref: 132 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.220079:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b04a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+872:3626:x1870152993274112:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.220082:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.220083:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02696c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x87d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.220091:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001c1885e3/0xd5401cbabc0221d6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x881:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28d4 expref: 132 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.220095:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02696c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x87d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.220096:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2179 count 1 00010000:00010000:0.0:1783520936.220108:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02696c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x87d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.220112:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350c5bb100/2, rc 0, transno 4294976442 00010000:00010000:2.0:1783520936.220113:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02699d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x883:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.220115:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001c1885e3/0xd5401cbabc0221d6 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x881:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28d4 expref: 132 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.220119:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02696c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x87d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.220125:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350c5bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993271936:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.220141:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x881:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.220143:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000cd47ee70/0xd5401cbabc0221eb lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x883:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f28f0 expref: 131 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.220146:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x881:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.220146:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.220155:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026996 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x881:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.220157:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x883:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.220166:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x883:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.220169:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.220171:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026996 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x881:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.220174:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02699d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x883:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.220175:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02696c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x87d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.220186:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993274240 00000100:00100000:3.0:1783520936.220192:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993274240 00010000:00010000:1.0:1783520936.220192:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.220193:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.220199:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02696c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x87d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.220206:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:0.0:1783520936.220209:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02696c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x87d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.220217:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b07b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993274368:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.220224:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02696c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x87d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.220225:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44947 00000100:00100000:1.0:1783520936.220225:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516310000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+131:2387:x1870153002479360:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 168us (555us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.220235:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc02696c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x87d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.220238:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36535 00000100:00100000:3.0:1783520936.220240:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b05c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+873:3626:x1870152993274240:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.220253:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.220257:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.220261:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.220264:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.220266:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993274368 00000100:00100000:1.0:1783520936.220270:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993274368 00002000:00080000:3.0:1783520936.220273:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2180 count 1 00010000:00010000:3.0:1783520936.220288:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0269a4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x884:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.220296:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44948 00010000:00010000:3.0:1783520936.220304:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005cecb8c6/0xd5401cbabc022525 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x884:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f28fe expref: 130 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.220306:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507938700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+874:3628:x1870152993274368:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.220319:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x884:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.220323:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x884:0x0].0x0 since=0s v=(0 0) 00002000:00080000:1.0:1783520936.220325:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2181 count 1 00010000:00010000:3.0:1783520936.220330:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0269a4 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x884:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.220339:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0269ab lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x885:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.220353:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d2c22d8c/0xd5401cbabc02251e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x885:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f290c expref: 130 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.220388:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350794ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+875:3628:x1870152993272320:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4829us (5267us total) trans 4294976443 rc 0/0 00010000:00010000:3.0:1783520936.220389:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005cecb8c6/0xd5401cbabc022525 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x884:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28fe expref: 130 pid: 35274 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.220395:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363fe54a80/2, rc 0, transno 4294976443 00000100:00100000:0.0:1783520936.220404:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44940 00000100:00100000:2.0:1783520936.220415:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363fe54a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993272320:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.220417:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005cecb8c6/0xd5401cbabc022525 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x884:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28fe expref: 130 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.220439:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000cd47ee70/0xd5401cbabc0221eb lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x883:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28f0 expref: 130 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.220450:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000cd47ee70/0xd5401cbabc0221eb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x883:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28f0 expref: 130 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.220460:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507939880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993274624:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.220474:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b07100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993274496:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.220516:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1780 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.220536:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993274752:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.220568:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.220575:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.220616:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026988 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x87f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.220635:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026988 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x87f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.220635:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x885:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.220642:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x885:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.220650:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0269ab lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x885:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.220652:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026988 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x87f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.220668:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026988 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x87f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.220669:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993274752 00000100:00100000:3.0:1783520936.220675:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993274752 00010000:00010000:1.0:1783520936.220681:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d2c22d8c/0xd5401cbabc02251e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x885:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f290c expref: 130 pid: 35276 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.220695:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d2c22d8c/0xd5401cbabc02251e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x885:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f290c expref: 130 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.220696:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520936.220706:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44949 00010000:00010000:2.0:1783520936.220710:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520936.220714:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ae4380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993274880:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.220716:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026988 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x87f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.220718:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350793aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+875:3628:x1870152993274752:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.220731:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.220735:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026988 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x87f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.220738:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2182 count 1 00010000:00010000:2.0:1783520936.220747:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026988 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x87f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.220752:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470f00 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.220753:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0269b2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x886:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.220759:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026988 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x87f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.220767:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026996 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x881:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.220773:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008e37893c/0xd5401cbabc02299a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x886:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f308a expref: 130 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.220777:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026988 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x87f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.220791:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026996 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x881:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.220797:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x886:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.220806:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x886:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.220810:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026996 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x881:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.220811:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.220812:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0269b2 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x886:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.220816:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.220820:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.220833:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026996 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x881:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.220869:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008e37893c/0xd5401cbabc02299a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x886:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f308a expref: 130 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.220880:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008e37893c/0xd5401cbabc02299a lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x886:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f308a expref: 130 pid: 12734 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.220890:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b3100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993275008:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.220947:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.220951:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.220956:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026996 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x881:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.220964:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.220968:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026996 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x881:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.220977:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026996 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x881:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.220989:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026996 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x881:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.220995:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026996 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x881:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.221007:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.221008:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.221009:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.221023:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516311c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+876:3627:x1870152993273728:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3265us (3372us total) trans 4294976444 rc 0/0 00000100:00100000:1.0:1783520936.221031:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44944 00000100:00100000:3.0:1783520936.221056:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3ef00 going to sleep for 11 seconds 00000004:00080000:0.0:1783520936.221056:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507939180/2, rc 0, transno 4294976444 00000100:00100000:0.0:1783520936.221072:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507939180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993273728:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.221094:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:2.0:1783520936.221101:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514fdc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+876:3626:x1870152993273216:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4478us (4574us total) trans 4294976445 rc 0/0 00010000:00010000:1.0:1783520936.221101:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02698f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x880:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.221118:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520936.221119:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44942 00010000:00010000:1.0:1783520936.221124:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02698f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x880:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.221131:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to regular blp list, 0 blwis in pool 00010000:00010000:1.0:1783520936.221135:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02698f lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x880:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.221141:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516313b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993275136:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.221154:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02698f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x880:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.221163:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc02698f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x880:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.221171:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.221186:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520936.221188:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363fe55c00/2, rc 0, transno 4294976445 00010000:00010000:1.0:1783520936.221190:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.221195:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.221203:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363fe55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993273216:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.221224:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0269b9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x882:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.221238:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000099790206/0xd5401cbabc0221e4 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x882:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f28e2 expref: 130 pid: 178496 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.221246:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x882:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.221250:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x882:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.221250:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993275136 00000100:00100000:0.0:1783520936.221255:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993275136 00010000:00010000:2.0:1783520936.221257:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0269b9 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x882:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.221278:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44950 00010000:00010000:2.0:1783520936.221283:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000099790206/0xd5401cbabc0221e4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x882:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28e2 expref: 130 pid: 178496 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.221287:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507938380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+876:3625:x1870152993275136:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.221300:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2183 count 1 00010000:00010000:2.0:1783520936.221301:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000099790206/0xd5401cbabc0221e4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x882:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28e2 expref: 130 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.221313:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350793bb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993275264:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.221316:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0269c0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x887:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.221337:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009aef09af/0xd5401cbabc02252c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x887:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2928 expref: 130 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.221341:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1180 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.221348:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x887:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.221352:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x887:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.221356:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0269c0 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x887:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.221429:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009aef09af/0xd5401cbabc02252c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x887:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2928 expref: 130 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.221440:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009aef09af/0xd5401cbabc02252c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x887:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2928 expref: 130 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.221450:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516310700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993275392:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.221500:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1783520936.221511:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993275520:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.221515:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80600 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.221539:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.221576:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507939880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993274624:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.221577:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993275520 00000100:00100000:0.0:1783520936.221584:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993275520 00010000:00010000:2.0:1783520936.221591:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02699d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x883:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.221621:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44951 00000100:00100000:0.0:1783520936.221632:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e472300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+877:3625:x1870152993275520:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.221644:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2184 count 1 00010000:00010000:0.0:1783520936.221655:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0269c7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x888:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.221667:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000909b6738/0xd5401cbabc022533 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x888:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2936 expref: 130 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.221693:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x888:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.221697:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x888:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.221701:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0269c7 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x888:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.221729:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000909b6738/0xd5401cbabc022533 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x888:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2936 expref: 130 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.221739:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000909b6738/0xd5401cbabc022533 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x888:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2936 expref: 130 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.221750:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507938a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+878:3625:x1870152993273472:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4703us (4811us total) trans 4294976446 rc 0/0 00000100:00100000:0.0:1783520936.221754:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350794df80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993275648:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.221763:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44943 00000004:00080000:2.0:1783520936.221772:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514fdd180/2, rc 0, transno 4294976446 00000100:00100000:2.0:1783520936.221785:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514fdd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993273472:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.221790:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80900 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.221829:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993275776:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.221855:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.221880:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993275776 00000100:00100000:2.0:1783520936.221885:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993275776 00000100:00100000:2.0:1783520936.221916:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44952 00000100:00100000:2.0:1783520936.221923:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435038ec380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+878:3627:x1870152993275776:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.221934:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2185 count 1 00010000:00010000:2.0:1783520936.221943:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0269ce lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x889:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.221952:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b75e3c8d/0xd5401cbabc02254f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x889:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2944 expref: 130 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.221956:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x889:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.221958:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x889:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.221960:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0269ce lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x889:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.221972:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b75e3c8d/0xd5401cbabc02254f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x889:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2944 expref: 130 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.221977:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b75e3c8d/0xd5401cbabc02254f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x889:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2944 expref: 130 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.221983:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e471f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993275904:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.222005:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1780 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.222809:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b07100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993274496:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.222829:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0269a4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x884:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.222844:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ae4380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993274880:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.222845:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.222863:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0269ab lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x885:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.222867:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b3100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993275008:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.222875:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.222891:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0269b2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x886:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.222903:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.222920:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002479488 00000100:00100000:1.0:1783520936.222926:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002479488 00000100:00100000:0.0:1783520936.222932:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002479616 00000100:00100000:2.0:1783520936.222935:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.222936:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002479616 00010000:00010000:1.0:1783520936.222948:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000cd47ee70/0xd5401cbabc0221eb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x883:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28f0 expref: 132 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.222956:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d2c22d8c/0xd5401cbabc02251e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x885:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f290c expref: 132 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.222975:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002479744 00000100:00100000:1.0:1783520936.222980:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002479744 00000100:00100000:2.0:1783520936.222990:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002479872 00010000:00010000:1.0:1783520936.222995:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005cecb8c6/0xd5401cbabc022525 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x884:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28fe expref: 133 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.222996:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002479872 00000100:00100000:0.0:1783520936.222996:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36536 00000100:00100000:0.0:1783520936.223007:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507939880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+134:2390:x1870153002479488:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.223014:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008e37893c/0xd5401cbabc02299a lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x886:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f308a expref: 134 pid: 12734 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.223021:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.223026:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36537 00010000:00010000:0.0:1783520936.223027:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000cd47ee70/0xd5401cbabc0221eb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x883:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28f0 expref: 134 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.223042:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36538 00000100:00100000:1.0:1783520936.223042:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee4000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+134:2388:x1870153002479616:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.223042:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000cd47ee70/0xd5401cbabc0221eb lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x883:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28f0 expref: 134 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.223053:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee4380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+134:2389:x1870153002479744:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.223058:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.223062:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.223064:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cd47ee70/0xd5401cbabc0221eb lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x883:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28f0 expref: 134 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.223067:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d2c22d8c/0xd5401cbabc02251e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x885:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f290c expref: 134 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.223069:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005cecb8c6/0xd5401cbabc022525 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x884:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28fe expref: 134 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.223081:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x883:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.223095:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x883:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.223100:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d2c22d8c/0xd5401cbabc02251e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x885:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f290c expref: 133 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.223101:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005cecb8c6/0xd5401cbabc022525 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x884:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28fe expref: 133 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.223102:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02699d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x883:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.223105:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36539 00000100:00100000:3.0:1783520936.223120:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94361b4a6850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+133:2389:x1870153002479872:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.223128:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02699d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x883:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.223133:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.223137:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005cecb8c6/0xd5401cbabc022525 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x884:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28fe expref: 133 pid: 35274 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.223141:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d2c22d8c/0xd5401cbabc02251e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x885:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f290c expref: 133 pid: 35276 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.223142:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008e37893c/0xd5401cbabc02299a lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x886:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f308a expref: 133 pid: 12734 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.223149:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.223160:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x884:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.223169:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x884:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.223173:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0269a4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x884:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.223174:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x885:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.223175:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008e37893c/0xd5401cbabc02299a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x886:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f308a expref: 131 pid: 12734 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.223179:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507939880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+131:2390:x1870153002479488:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 172us (332us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.223184:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x885:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.223196:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0269a4 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x884:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.223196:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0269ab lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x885:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.223198:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36536 00010000:00010000:3.0:1783520936.223208:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008e37893c/0xd5401cbabc02299a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x886:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 20971520->20975615) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f308a expref: 131 pid: 12734 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.223222:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0269ab lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x885:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.223229:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x886:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.223232:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.223234:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x886:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.223241:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0269b2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x886:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.223250:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0269b2 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x886:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.223251:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.223253:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee4380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+129:2389:x1870153002479744:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 200us (350us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.223259:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0269ab lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x885:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.223274:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36538 00010000:00010000:0.0:1783520936.223276:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0269ab lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x885:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.223281:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee4000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+129:2388:x1870153002479616:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 243us (407us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.223287:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.223297:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36537 00010000:00010000:0.0:1783520936.223297:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0269ab lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x885:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.223309:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0269ab lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x885:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.223316:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94361b4a6850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+127:2389:x1870153002479872:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 198us (378us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.223328:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36539 00010000:00010000:0.0:1783520936.223331:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.223342:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.223347:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0269ab lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x885:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.223355:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbbe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.223380:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0269ab lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x885:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.223389:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0269ab lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x885:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.223400:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02699d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x883:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.223401:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0269ab lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x885:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.223405:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0269b2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x886:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.223416:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02699d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x883:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.223417:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc0269ab lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x885:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.223424:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02699d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x883:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.223431:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02699d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x883:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.223436:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.223438:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.223440:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.223444:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to regular blp list, 0 blwis in pool 00010000:00010000:0.0:1783520936.223449:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02699d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x883:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.223452:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.223469:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.223474:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0269b2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x886:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.223476:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02699d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x883:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.223504:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0269b2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x886:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.223522:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0269b2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x886:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.223523:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02699d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x883:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.223545:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02699d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x883:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.223553:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.223553:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc02699d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x883:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.223554:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b04a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+879:3626:x1870152993274112:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3481us (3664us total) trans 4294976447 rc 0/0 00000100:00100000:2.0:1783520936.223566:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44946 00010000:00010000:0.0:1783520936.223575:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.223579:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0269b2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x886:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.223582:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513b04e00/2, rc 0, transno 4294976447 00010000:00010000:2.0:1783520936.223585:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.223594:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4c00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:3.0:1783520936.223598:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b04e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993274112:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.223598:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0269b2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x886:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.223600:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0269a4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x884:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.223613:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0269a4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x884:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.223613:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0269b2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x886:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.223622:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0269a4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x884:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.223625:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0269b2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x886:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.223632:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0269a4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x884:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.223633:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc0269b2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x886:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.223648:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.223649:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.223652:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0269a4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x884:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.223658:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.223662:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0269a4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x884:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.223666:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.223678:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0269a4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x884:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.223681:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993276032:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.223695:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0269a4 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x884:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.223704:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc0269a4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x884:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.223719:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.223723:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.223725:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.223726:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.223741:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b05c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+879:3626:x1870152993274240:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3505us (3664us total) trans 4294976448 rc 0/0 00000100:00100000:2.0:1783520936.223748:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44947 00000100:00100000:2.0:1783520936.223769:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993276032 00000100:00100000:2.0:1783520936.223773:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993276032 00000004:00080000:3.0:1783520936.223781:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513b06d80/2, rc 0, transno 4294976448 00000100:00100000:3.0:1783520936.223796:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b06d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993274240:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.223801:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44953 00000100:00100000:2.0:1783520936.223810:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ae5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+880:3628:x1870152993276032:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.223823:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2186 count 1 00000100:00100000:0.0:1783520936.223830:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507938700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+880:3628:x1870152993274368:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3525us (3578us total) trans 4294976449 rc 0/0 00010000:00010000:2.0:1783520936.223835:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0269d5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x88a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.223837:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44948 00000100:00100000:3.0:1783520936.223845:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:2.0:1783520936.223848:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006a1c5e72/0xd5401cbabc022541 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x88a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2952 expref: 126 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.223857:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993276160:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.223858:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x88a:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.223866:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x88a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.223871:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0269d5 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x88a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.223871:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513b07b80/2, rc 0, transno 4294976449 00000100:00100000:1.0:1783520936.223888:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b07b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993274368:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.223900:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.223910:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006a1c5e72/0xd5401cbabc022541 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x88a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2952 expref: 126 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.223910:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350794df80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993275648:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.223927:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006a1c5e72/0xd5401cbabc022541 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x88a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2952 expref: 126 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.223931:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0269c7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x888:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.223937:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ae4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993276416:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.223940:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993276288:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.223990:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1000 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.224145:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.224191:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993276160 00000100:00100000:3.0:1783520936.224192:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993276416 00000100:00100000:3.0:1783520936.224199:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993276416 00000100:00100000:0.0:1783520936.224206:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993276160 00000100:00100000:3.0:1783520936.224235:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44954 00000100:00100000:3.0:1783520936.224247:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+881:3626:x1870152993276416:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.224253:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44955 00000100:00100000:2.0:1783520936.224254:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.224256:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350793bb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993275264:192.168.204.33@tcp:104: 00002000:00080000:3.0:1783520936.224268:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2188 count 1 00000100:00100000:0.0:1783520936.224270:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363573b450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+881:3626:x1870152993276160:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.224282:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.224283:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0269b9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x882:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.224285:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0269dc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x88c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.224286:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2187 count 1 00010000:00010000:0.0:1783520936.224301:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0269e3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x88b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.224302:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.224317:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f98776dc/0xd5401cbabc022548 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x88c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f296e expref: 126 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.224319:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000055050f2c/0xd5401cbabc02253a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x88b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2960 expref: 126 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.224329:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.224340:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x88b:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.224343:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x88c:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.224345:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x88b:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.224349:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002480128 00010000:00010000:3.0:1783520936.224350:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x88c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.224351:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0269e3 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x88b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.224354:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002480128 00010000:00010000:2.0:1783520936.224386:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000099790206/0xd5401cbabc0221e4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x882:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28e2 expref: 127 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.224398:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0269dc lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x88c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.224398:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000055050f2c/0xd5401cbabc02253a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x88b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2960 expref: 127 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.224401:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002480384 00000100:00100000:2.0:1783520936.224404:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002480384 00010000:00010000:2.0:1783520936.224416:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b75e3c8d/0xd5401cbabc02254f lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x889:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2944 expref: 128 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.224429:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000055050f2c/0xd5401cbabc02253a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x88b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2960 expref: 128 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.224434:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002480256 00000100:00100000:2.0:1783520936.224444:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002480256 00000100:00100000:0.0:1783520936.224444:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350794df80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993276544:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.224458:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f98776dc/0xd5401cbabc022548 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x88c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f296e expref: 129 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.224467:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009aef09af/0xd5401cbabc02252c lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x887:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2928 expref: 129 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.224475:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f98776dc/0xd5401cbabc022548 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x88c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f296e expref: 129 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.224503:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002480000 00000100:00100000:3.0:1783520936.224504:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516310000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993276672:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.224508:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80900 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.224509:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002480000 00010000:00010000:2.0:1783520936.224530:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000909b6738/0xd5401cbabc022533 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x888:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2936 expref: 130 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.224538:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e840 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.224549:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36540 00000100:00100000:2.0:1783520936.224558:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee4000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+130:2388:x1870153002480128:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520936.224561:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36541 00010000:00010000:2.0:1783520936.224570:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.224576:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350544e300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+130:2388:x1870153002480384:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.224579:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000099790206/0xd5401cbabc0221e4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x882:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28e2 expref: 130 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.224591:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000099790206/0xd5401cbabc0221e4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x882:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28e2 expref: 130 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.224605:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000099790206/0xd5401cbabc0221e4 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x882:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f28e2 expref: 130 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.224616:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x882:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.224616:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516310700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993275392:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.224622:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x882:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.224628:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0269b9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x882:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.224635:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0269c0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x887:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.224637:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0269b9 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x882:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.224667:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.224695:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee4000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+129:2388:x1870153002480128:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 138us (444us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.224704:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36540 00000100:00100000:1.0:1783520936.224716:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36542 00000100:00100000:2.0:1783520936.224726:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36543 00000100:00100000:1.0:1783520936.224732:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350544d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+128:2390:x1870153002480256:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.224738:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513139c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+128:2390:x1870153002480000:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.224743:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.224745:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.224754:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000909b6738/0xd5401cbabc022533 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x888:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2936 expref: 128 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.224755:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009aef09af/0xd5401cbabc02252c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x887:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2928 expref: 128 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.224775:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000909b6738/0xd5401cbabc022533 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x888:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2936 expref: 128 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.224783:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009aef09af/0xd5401cbabc02252c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x887:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2928 expref: 128 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.224788:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000909b6738/0xd5401cbabc022533 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x888:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2936 expref: 128 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.224804:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x888:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.224807:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x888:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.224813:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009aef09af/0xd5401cbabc02252c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x887:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2928 expref: 127 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.224814:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0269c7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x888:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.224837:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0269c7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x888:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.224841:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x887:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.224849:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x887:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.224855:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0269c0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x887:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.224863:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.224866:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0269c0 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x887:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.224887:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0269c7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x888:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.224890:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513139c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+126:2390:x1870153002480000:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 154us (564us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.224893:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.224898:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0269c7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x888:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.224899:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36543 00010000:00010000:0.0:1783520936.224907:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0269c7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x888:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.224915:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0269c7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x888:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.224924:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350544d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+125:2390:x1870153002480256:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 197us (622us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.224934:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36542 00010000:00010000:0.0:1783520936.224937:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.224966:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0269b9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x882:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.224977:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0269b9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x882:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.224987:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0269b9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x882:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.224995:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0269b9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x882:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.225013:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520936.225015:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520936.225022:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0269c7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x888:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.225025:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e472300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+882:3625:x1870152993275520:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3396us (3485us total) trans 4294976450 rc 0/0 00010000:00010000:2.0:1783520936.225032:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea200 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520936.225035:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44951 00010000:00010000:2.0:1783520936.225037:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0269c7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x888:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.225042:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000004:00080000:1.0:1783520936.225044:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b1180/2, rc 0, transno 4294976450 00010000:00010000:2.0:1783520936.225046:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0269c7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x888:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.225052:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b75e3c8d/0xd5401cbabc02254f lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x889:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2944 expref: 124 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.225058:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993275520:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.225064:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0269c7 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x888:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.225080:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b75e3c8d/0xd5401cbabc02254f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x889:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2944 expref: 124 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.225081:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cfc011f5/0xd5401cbabc0269c7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x888:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.225115:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x889:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.225115:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520936.225115:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:3.0:1783520936.225124:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x889:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.225124:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0269b9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x882:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.225126:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0269c0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x887:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.225130:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503372a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993276800:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.225133:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0269ce lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x889:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.225143:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.225144:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0269c0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x887:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.225148:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0269b9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x882:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.225162:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0269b9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x882:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.225162:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0269c0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x887:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.225174:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.225175:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0269c0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x887:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.225185:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0269b9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x882:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.225193:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc0269b9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x882:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.225195:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.225196:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0269ce lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x889:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.225205:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.225212:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.225212:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0269c0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x887:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.225216:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.225218:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.225220:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.225226:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.225229:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.225232:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0269c0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x887:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.225243:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0269c0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x887:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.225249:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350544e300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+124:2388:x1870153002480384:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 676us (965us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.225261:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36541 00010000:00010000:1.0:1783520936.225263:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0269c0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x887:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.225274:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc0269c0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x887:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.225274:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507938380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+882:3625:x1870152993275136:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3986us (4103us total) trans 4294976451 rc 0/0 00000100:00100000:3.0:1783520936.225288:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993276800 00000100:00100000:0.0:1783520936.225292:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44950 00000100:00100000:3.0:1783520936.225295:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993276800 00010000:00010000:1.0:1783520936.225305:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.225309:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.225312:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.225336:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44956 00000100:00100000:2.0:1783520936.225337:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000004:00080000:1.0:1783520936.225341:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516313b80/2, rc 0, transno 4294976451 00000100:00100000:0.0:1783520936.225347:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503370e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+882:3628:x1870152993276800:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.225379:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002480512 00000100:00100000:1.0:1783520936.225383:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516313b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993275136:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.225384:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002480512 00002000:00080000:0.0:1783520936.225391:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2189 count 1 00010000:00010000:2.0:1783520936.225402:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006a1c5e72/0xd5401cbabc022541 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x88a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2952 expref: 124 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.225404:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0269ea lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x88d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.225418:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000004b72886f/0xd5401cbabc022556 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x88d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f297c expref: 124 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.225426:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36544 00010000:00010000:0.0:1783520936.225426:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x88d:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.225430:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x88d:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.225433:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1783520936.225434:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351313b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+124:2389:x1870153002480512:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.225435:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0269ea lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x88d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.225445:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.225447:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503373480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993276928:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.225453:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006a1c5e72/0xd5401cbabc022541 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x88a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2952 expref: 124 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.225458:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000004b72886f/0xd5401cbabc022556 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x88d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f297c expref: 124 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.225466:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006a1c5e72/0xd5401cbabc022541 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x88a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2952 expref: 124 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.225468:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000004b72886f/0xd5401cbabc022556 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x88d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f297c expref: 124 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.225481:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502250e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993277056:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.225486:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x88a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.225504:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x88a:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.225508:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.225510:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350793aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+883:3628:x1870152993274752:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4792us (4933us total) trans 4294976452 rc 0/0 00010000:00010000:2.0:1783520936.225514:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0269d5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.225527:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44949 00010000:00010000:2.0:1783520936.225529:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0269d5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x88a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.225539:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80e40 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.225544:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.225564:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351313b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+124:2389:x1870153002480512:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 131us (226us total) trans 0 rc 0/0 00000004:00080000:1.0:1783520936.225567:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b1f80/2, rc 0, transno 4294976452 00000100:00100000:2.0:1783520936.225573:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36544 00000100:00100000:1.0:1783520936.225580:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993274752:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.225600:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993276928 00000100:00100000:2.0:1783520936.225604:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993276928 00000100:00100000:1.0:1783520936.225628:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503371500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993277184:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.225652:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44957 00000100:00100000:2.0:1783520936.225660:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503372300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+883:3627:x1870152993276928:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.225677:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00080000:2.0:1783520936.225684:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2190 count 1 00010000:00010000:2.0:1783520936.225698:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0269f1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x88e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.225712:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d932c215/0xd5401cbabc022889 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x88e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f79 expref: 123 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.225719:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x88e:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.225722:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x88e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.225727:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0269f1 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x88e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.225727:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993277184 00000100:00100000:0.0:1783520936.225727:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350794df80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993276544:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.225734:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993277184 00010000:00010000:0.0:1783520936.225749:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0269e3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x88b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.225752:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d932c215/0xd5401cbabc022889 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x88e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f79 expref: 123 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.225763:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d932c215/0xd5401cbabc022889 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x88e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f79 expref: 123 pid: 35274 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.225774:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516310000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993276672:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.225774:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44958 00000100:00100000:2.0:1783520936.225775:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513138000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993277312:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.225791:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+884:3625:x1870152993277184:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.225791:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.225806:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1480 going to sleep for 11 seconds 00002000:00080000:1.0:1783520936.225807:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2191 count 1 00000100:00100000:2.0:1783520936.225822:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002480640 00010000:00010000:1.0:1783520936.225825:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0269f8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x88f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.225826:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002480640 00010000:00010000:2.0:1783520936.225843:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000055050f2c/0xd5401cbabc02253a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x88b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2960 expref: 124 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.225848:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000bfa1c2ad/0xd5401cbabc02256b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x88f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2998 expref: 124 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.225862:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x88f:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.225866:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x88f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.225871:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0269f8 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x88f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.225876:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36545 00000100:00100000:2.0:1783520936.225887:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350794df80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+124:2388:x1870153002480640:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.225896:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.225902:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000055050f2c/0xd5401cbabc02253a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x88b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2960 expref: 124 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.225905:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000bfa1c2ad/0xd5401cbabc02256b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x88f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2998 expref: 124 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.225922:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000055050f2c/0xd5401cbabc02253a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x88b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2960 expref: 124 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.225938:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000bfa1c2ad/0xd5401cbabc02256b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x88f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2998 expref: 124 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.225945:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000055050f2c/0xd5401cbabc02253a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x88b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2960 expref: 124 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.225955:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350d166a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993277440:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.225958:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x88b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.225962:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x88b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.225965:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0269e3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.225971:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0269e3 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x88b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.225989:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.225990:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470780 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.226015:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350794df80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+123:2388:x1870153002480640:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 128us (226us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.226023:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36545 00010000:00010000:3.0:1783520936.226047:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0269dc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x88c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.226066:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b75e3c8d/0xd5401cbabc02254f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x889:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2944 expref: 122 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.226074:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.226079:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e471f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993275904:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.226094:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0269ce lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x889:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.226099:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0269e3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.226103:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0269ce lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x889:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.226108:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516311880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+885:3627:x1870152993273984:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6097us (6346us total) trans 4294976453 rc 0/0 00010000:00010000:2.0:1783520936.226116:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0269ce lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x889:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.226120:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0269e3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.226126:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44945 00010000:00010000:0.0:1783520936.226134:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0269e3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.226138:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0269ce lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x889:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.226141:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0269e3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x88b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.226146:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0269ce lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x889:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.226155:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000004:00080000:1.0:1783520936.226165:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435079f2a00/2, rc 0, transno 4294976453 00010000:00010000:2.0:1783520936.226167:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:0.0:1783520936.226168:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520936.226174:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0269e3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x88b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.226179:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435079f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993273984:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.226188:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.226193:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0269e3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x88b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.226207:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0269e3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x88b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.226221:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0269e3 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x88b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.226228:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc0269e3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x88b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.226232:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993277568:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.226239:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435038ec380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+885:3627:x1870152993275776:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4315us (4381us total) trans 4294976454 rc 0/0 00000100:00100000:2.0:1783520936.226247:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44952 00010000:00010000:0.0:1783520936.226251:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.226254:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0269ce lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x889:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.226260:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.226264:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0269ce lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x889:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.226273:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.226276:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0269ce lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x889:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.226288:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0269ce lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x889:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.226294:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc0269ce lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x889:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.226312:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.226313:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.226316:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520936.226326:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b0380/2, rc 0, transno 4294976454 00000100:00100000:1.0:1783520936.226337:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993275776:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.226381:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.226405:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1783520936.226417:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993277696:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.226419:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363573b450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+885:3626:x1870152993276160:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2153us (2520us total) trans 4294976455 rc 0/0 00000100:00100000:0.0:1783520936.226429:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44955 00000100:00100000:0.0:1783520936.226441:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993277568 00000100:00100000:0.0:1783520936.226445:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993277568 00000100:00100000:1.0:1783520936.226455:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.226468:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993277696 00000100:00100000:0.0:1783520936.226473:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993277696 00010000:00010000:2.0:1783520936.226503:0:35282:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006a1c5e72/0xd5401cbabc022541 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x88a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2952 expref: 121 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.226510:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44959 00000100:00100000:1.0:1783520936.226517:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002480768 00000100:00100000:2.0:1783520936.226518:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993276288:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.226518:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513bc0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+886:3626:x1870152993277568:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.226525:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002480768 00002000:00080000:0.0:1783520936.226533:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2192 count 1 00010000:00010000:2.0:1783520936.226537:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0269d5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.226545:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0269d5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.226546:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0269ff lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x890:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.226548:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f98776dc/0xd5401cbabc022548 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x88c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f296e expref: 121 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.226559:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0269d5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.226563:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e31756c3/0xd5401cbabc022564 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x890:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f29a6 expref: 121 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.226571:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x890:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.226572:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0269d5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.226575:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x890:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.226579:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0269ff lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x890:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.226580:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0269d5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x88a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.226585:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36546 00000100:00100000:1.0:1783520936.226598:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f0c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+121:2388:x1870153002480768:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.226601:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.226608:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.226608:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e31756c3/0xd5401cbabc022564 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x890:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29a6 expref: 121 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.226610:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.226616:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0269d5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x88a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.226618:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f98776dc/0xd5401cbabc022548 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x88c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f296e expref: 121 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.226622:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e31756c3/0xd5401cbabc022564 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x890:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29a6 expref: 121 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.226628:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.226633:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0269d5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x88a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.226648:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0269d5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x88a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.226649:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993277824:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.226653:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f98776dc/0xd5401cbabc022548 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x88c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f296e expref: 121 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.226663:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0269d5 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x88a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.226672:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc0269d5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x88a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.226676:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f98776dc/0xd5401cbabc022548 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x88c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f296e expref: 121 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.226683:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80900 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.226693:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.226696:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.226698:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.226699:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x88c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.226704:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x88c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.226709:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0269dc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.226718:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0269dc lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x88c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.226719:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44960 00000100:00100000:0.0:1783520936.226730:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507acf100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+887:3628:x1870152993277696:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.226742:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00002000:00080000:0.0:1783520936.226743:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2193 count 1 00010000:00010000:0.0:1783520936.226757:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a06 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x891:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.226768:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b42dfda1/0xd5401cbabc022572 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x891:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f29b4 expref: 120 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.226771:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ae5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+887:3628:x1870152993276032:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2962us (3049us total) trans 4294976456 rc 0/0 00010000:00010000:0.0:1783520936.226776:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x891:0x0].0x0 since=38s v=(0 0) 00000100:00100000:1.0:1783520936.226779:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f0c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+120:2388:x1870153002480768:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 181us (702us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.226780:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x891:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.226783:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44953 00010000:00010000:0.0:1783520936.226788:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a06 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x891:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.226795:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36546 00010000:00010000:0.0:1783520936.226814:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b42dfda1/0xd5401cbabc022572 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x891:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29b4 expref: 119 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.226825:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b42dfda1/0xd5401cbabc022572 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x891:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29b4 expref: 119 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.226834:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530d500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993277952:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.226861:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80540 going to sleep for 11 seconds 00000004:00080000:1.0:1783520936.226878:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b3100/2, rc 0, transno 4294976456 00000100:00100000:1.0:1783520936.226890:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993276032:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.226937:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515477b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993278080:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.226963:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.226980:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993278080 00000100:00100000:1.0:1783520936.226984:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993278080 00000100:00100000:2.0:1783520936.227010:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44961 00000100:00100000:2.0:1783520936.227019:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351313a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+887:3628:x1870152993278080:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.227030:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2194 count 1 00010000:00010000:2.0:1783520936.227041:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026a0d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x892:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.227051:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000082e41d76/0xd5401cbabc0225a3 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x892:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f29c2 expref: 119 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.227058:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x892:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.227061:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x892:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.227065:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026a0d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x892:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.227083:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000082e41d76/0xd5401cbabc0225a3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x892:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29c2 expref: 119 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.227093:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000082e41d76/0xd5401cbabc0225a3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x892:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29c2 expref: 119 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.227103:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351313b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993278208:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.227138:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1000 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.227182:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502250e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993277056:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.227196:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0269ea lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x88d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.227434:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b1880/2, rc 0, transno 4294976455 00000100:00100000:2.0:1783520936.227443:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993276160:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.227473:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.227508:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002480896 00000100:00100000:0.0:1783520936.227511:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515475500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993278336:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.227514:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002480896 00010000:00010000:3.0:1783520936.227539:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000004b72886f/0xd5401cbabc022556 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x88d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f297c expref: 120 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.227550:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350d166a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993277440:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.227559:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.227567:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0269f8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x88f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.227581:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.227588:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36547 00000100:00100000:3.0:1783520936.227599:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502250e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+120:2390:x1870153002480896:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.227601:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513138000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993277312:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.227615:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0269f1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x88e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.227626:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.227627:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993278336 00000100:00100000:0.0:1783520936.227634:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993278336 00010000:00010000:3.0:1783520936.227637:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000004b72886f/0xd5401cbabc022556 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x88d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f297c expref: 120 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.227639:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002481024 00000100:00100000:2.0:1783520936.227646:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002481024 00000100:00100000:1.0:1783520936.227648:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.227660:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000004b72886f/0xd5401cbabc022556 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x88d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f297c expref: 121 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.227668:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000bfa1c2ad/0xd5401cbabc02256b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x88f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2998 expref: 121 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.227681:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000004b72886f/0xd5401cbabc022556 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x88d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f297c expref: 121 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.227682:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44962 00000100:00100000:0.0:1783520936.227694:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515474700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+888:3625:x1870152993278336:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.227696:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002481152 00000100:00100000:2.0:1783520936.227702:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002481152 00002000:00080000:0.0:1783520936.227711:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2195 count 1 00010000:00010000:2.0:1783520936.227719:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d932c215/0xd5401cbabc022889 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x88e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f79 expref: 122 pid: 35274 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.227727:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026a14 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x893:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.227743:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d45249a2/0xd5401cbabc022579 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x893:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f29d0 expref: 121 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.227748:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36548 00010000:00010000:0.0:1783520936.227757:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x893:0x0].0x0 since=38s v=(0 0) 00000100:00100000:1.0:1783520936.227759:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36549 00000100:00100000:2.0:1783520936.227763:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350544d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+121:2388:x1870153002481024:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.227766:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x893:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.227774:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026a14 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x893:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.227777:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.227777:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350544e300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+121:2390:x1870153002481152:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.227786:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000bfa1c2ad/0xd5401cbabc02256b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x88f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2998 expref: 121 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.227795:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.227806:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d932c215/0xd5401cbabc022889 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x88e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f79 expref: 121 pid: 35274 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.227809:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d45249a2/0xd5401cbabc022579 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x893:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29d0 expref: 121 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.227812:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000bfa1c2ad/0xd5401cbabc02256b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x88f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2998 expref: 121 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.227840:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d932c215/0xd5401cbabc022889 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x88e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f79 expref: 121 pid: 35274 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.227843:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000bfa1c2ad/0xd5401cbabc02256b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x88f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2998 expref: 121 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.227844:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x88d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.227845:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d45249a2/0xd5401cbabc022579 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x893:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29d0 expref: 121 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.227856:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x88d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.227869:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0269ea lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.227872:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x88f:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.227878:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515476680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993278464:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.227880:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x88f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.227880:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d932c215/0xd5401cbabc022889 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x88e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f79 expref: 120 pid: 35274 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.227890:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0269ea lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x88d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.227892:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0269f8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.227900:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x88e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.227909:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x88e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.227917:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0269f8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x88f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.227919:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0269f1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.227930:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80300 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.227932:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0269f1 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x88e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.227935:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.227941:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.227956:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.227966:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502250e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+119:2390:x1870153002480896:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 368us (492us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.227974:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350544d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+119:2388:x1870153002481024:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 214us (393us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.227980:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36547 00000100:00100000:1.0:1783520936.227986:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350544e300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+119:2390:x1870153002481152:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 211us (340us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.227992:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36548 00000100:00100000:1.0:1783520936.228001:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36549 00010000:00010000:3.0:1783520936.228023:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0269dc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.228024:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0269f1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.228031:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0269ea lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.228045:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0269dc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.228048:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0269f1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.228056:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0269ea lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.228071:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0269dc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.228076:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0269f1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.228082:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0269ea lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.228096:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0269dc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x88c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.228104:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0269f1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x88e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.228110:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0269ea lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x88d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.228132:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520936.228141:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:0.0:1783520936.228153:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520936.228159:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0269f1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x88e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.228160:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.228169:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0269ea lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x88d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.228173:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.228180:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0269f1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x88e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.228181:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.228200:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0269ea lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x88d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.228206:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0269f1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x88e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.228210:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0269ea lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x88d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.228221:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503372300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+890:3627:x1870152993276928:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2560us (2728us total) trans 4294976457 rc 0/0 00010000:00010000:0.0:1783520936.228231:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0269f1 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x88e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.228234:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44957 00010000:00010000:3.0:1783520936.228239:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0269ea lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x88d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.228239:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503373480/2, rc 0, transno 4294976457 00010000:00010000:0.0:1783520936.228252:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc0269f1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x88e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.228254:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0269f8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.228255:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000958abacc/0xd5401cbabc0269ea lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x88d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.228257:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503373480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993276928:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.228281:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0269f8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.228293:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0269f8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x88f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.228296:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.228298:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.228301:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.228302:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.228305:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.228308:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.228309:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0269f8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x88f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.228322:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1783520936.228332:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350794ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993278592:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.228333:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520936.228337:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1783520936.228347:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1783520936.228351:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0269dc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x88c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.228385:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.228389:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0269dc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x88c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.228398:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0269dc lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x88c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.228399:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.228404:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.228412:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0269dc lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x88c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.228412:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0269f8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x88f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.228431:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc0269dc lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x88c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.228434:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbaa00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.228441:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0269f8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x88f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.228447:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.228451:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.228452:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0269f8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x88f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.228453:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.228458:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503370e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+889:3628:x1870152993276800:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3113us (3287us total) trans 4294976458 rc 0/0 00000100:00100000:1.0:1783520936.228472:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44956 00000004:00080000:2.0:1783520936.228477:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503372a00/2, rc 0, transno 4294976458 00010000:00010000:0.0:1783520936.228477:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0269f8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x88f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.228504:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503372a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993276800:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.228504:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993278592 00000100:00100000:1.0:1783520936.228511:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993278592 00010000:00010000:0.0:1783520936.228517:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc0269f8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x88f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.228537:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+890:3626:x1870152993276416:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4291us (4557us total) trans 4294976459 rc 0/0 00010000:00010000:0.0:1783520936.228540:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.228544:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.228547:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44963 00010000:00010000:0.0:1783520936.228550:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.228556:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44954 00000100:00100000:2.0:1783520936.228557:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1783520936.228571:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513138000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993278720:0@lo:6:osp-syn-1-0.0 00000004:00080000:3.0:1783520936.228590:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ae4380/2, rc 0, transno 4294976459 00000100:00100000:2.0:1783520936.228609:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.228613:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ae4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993276416:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.228626:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993277824:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.228640:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993278720 00000100:00100000:3.0:1783520936.228646:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993278720 00010000:00010000:0.0:1783520936.228649:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0269ff lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x890:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.228672:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.228678:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351313b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993278848:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.228691:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530d500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993277952:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.228703:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.228709:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a06 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x891:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.228712:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.228738:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002481408 00000100:00100000:0.0:1783520936.228742:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002481408 00000100:00100000:1.0:1783520936.228746:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.228755:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993278848 00000100:00100000:2.0:1783520936.228759:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993278848 00000100:00100000:3.0:1783520936.228764:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44964 00010000:00010000:0.0:1783520936.228766:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b42dfda1/0xd5401cbabc022572 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x891:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29b4 expref: 117 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.228778:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513139c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+891:3628:x1870152993278720:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.228783:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44965 00000100:00100000:2.0:1783520936.228797:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350d166a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+891:3628:x1870152993278848:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.228797:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002481280 00002000:00080000:3.0:1783520936.228800:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2197 count 1 00000100:00100000:0.0:1783520936.228804:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002481280 00000100:00100000:1.0:1783520936.228812:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002481536 00002000:00080000:2.0:1783520936.228816:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2198 count 1 00000100:00100000:1.0:1783520936.228817:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002481536 00010000:00010000:3.0:1783520936.228819:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026a1b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x895:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.228824:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e31756c3/0xd5401cbabc022564 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x890:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29a6 expref: 119 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.228836:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002540c41a/0xd5401cbabc022587 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x895:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f29ec expref: 119 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.228836:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000082e41d76/0xd5401cbabc0225a3 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x892:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29c2 expref: 119 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.228837:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026a22 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x896:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.228853:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36550 00000100:00100000:0.0:1783520936.228867:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351635b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+119:2389:x1870153002481408:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.228872:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e719db35/0xd5401cbabc022ae3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x896:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f31d3 expref: 119 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.228876:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.228883:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x896:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.228885:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b42dfda1/0xd5401cbabc022572 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x891:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29b4 expref: 119 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.228887:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x896:0x0].0x0 since=0s v=(1 0) 00000100:00100000:1.0:1783520936.228890:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36551 00010000:00010000:2.0:1783520936.228897:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026a22 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x896:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.228903:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x895:0x0].0x0 since=38s v=(0 0) 00000100:00100000:1.0:1783520936.228907:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514dd6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+119:2389:x1870153002481280:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.228914:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x895:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.228918:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b42dfda1/0xd5401cbabc022572 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x891:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29b4 expref: 119 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.228924:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026a1b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x895:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.228926:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.228934:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e719db35/0xd5401cbabc022ae3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x896:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31d3 expref: 119 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.228938:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e31756c3/0xd5401cbabc022564 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x890:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29a6 expref: 119 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.228943:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b42dfda1/0xd5401cbabc022572 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x891:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29b4 expref: 119 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.228965:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e719db35/0xd5401cbabc022ae3 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x896:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31d3 expref: 119 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.228975:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e31756c3/0xd5401cbabc022564 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x890:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29a6 expref: 118 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.228976:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x891:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.228986:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x891:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.228987:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363d3a6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993278976:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.228999:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a06 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x891:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.229007:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e31756c3/0xd5401cbabc022564 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x890:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29a6 expref: 118 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.229012:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a06 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x891:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.229026:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x890:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.229031:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x890:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.229034:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1cc0 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.229037:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0269ff lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x890:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.229040:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.229046:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002540c41a/0xd5401cbabc022587 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x895:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29ec expref: 117 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.229053:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36552 00000100:00100000:2.0:1783520936.229068:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514dd7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+117:2389:x1870153002481536:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.229069:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351635b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+117:2389:x1870153002481408:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 200us (393us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.229073:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0269ff lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x890:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.229076:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002540c41a/0xd5401cbabc022587 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x895:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29ec expref: 117 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.229079:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.229088:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000082e41d76/0xd5401cbabc0225a3 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x892:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29c2 expref: 117 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.229089:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36550 00000100:00100000:3.0:1783520936.229111:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516310000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993279104:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.229111:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000082e41d76/0xd5401cbabc0225a3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x892:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29c2 expref: 116 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.229118:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.229130:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x892:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.229133:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x892:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.229136:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026a0d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x892:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.229141:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026a0d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x892:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.229151:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514dd6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+116:2389:x1870153002481280:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 246us (445us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.229151:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.229152:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.229155:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.229167:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36551 00000100:00100000:3.0:1783520936.229173:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002481664 00000100:00100000:3.0:1783520936.229177:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002481664 00000100:00100000:2.0:1783520936.229179:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514dd7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+116:2389:x1870153002481536:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 115us (432us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.229189:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36552 00010000:00010000:3.0:1783520936.229196:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d45249a2/0xd5401cbabc022579 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x893:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29d0 expref: 116 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.229207:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350794df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+891:3625:x1870152993278592:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.229213:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36553 00002000:00080000:1.0:1783520936.229222:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2196 count 1 00000100:00100000:2.0:1783520936.229224:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f0450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+115:2389:x1870153002481664:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.229235:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.229239:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a29 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x894:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.229240:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d45249a2/0xd5401cbabc022579 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x893:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29d0 expref: 115 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.229267:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d45249a2/0xd5401cbabc022579 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x893:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29d0 expref: 115 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.229267:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002afd2964/0xd5401cbabc022580 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x894:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f29de expref: 115 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.229290:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x894:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.229294:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x894:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.229298:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x893:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.229299:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a29 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x894:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.229303:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a06 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x891:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.229306:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x893:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.229316:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026a14 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x893:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.229343:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002afd2964/0xd5401cbabc022580 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x894:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29de expref: 115 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.229343:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a06 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x891:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.229344:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026a14 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x893:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.229382:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a06 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x891:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.229399:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.229405:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a06 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x891:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.229412:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002afd2964/0xd5401cbabc022580 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x894:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29de expref: 115 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.229426:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514dd7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993279232:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.229427:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.229430:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f0450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+115:2389:x1870153002481664:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 207us (280us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.229442:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36553 00010000:00010000:0.0:1783520936.229442:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.229449:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a06 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x891:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.229459:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.229463:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a06 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x891:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.229474:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a06 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x891:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.229486:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470a80 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.229497:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a06 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x891:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.229509:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a06 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x891:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.229531:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.229534:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.229536:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.229541:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000082e41d76/0xd5401cbabc0225a3 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x892:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29c2 expref: 114 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.229542:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+892:3625:x1870152993277184:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3755us (3864us total) trans 4294976460 rc 0/0 00000004:00080000:1.0:1783520936.229548:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503371500/2, rc 0, transno 4294976460 00000100:00100000:3.0:1783520936.229560:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44958 00000100:00100000:2.0:1783520936.229564:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351313b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993278208:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.229572:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503371500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993277184:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.229580:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026a0d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x892:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.229631:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507acf100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+892:3628:x1870152993277696:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2901us (3175us total) trans 4294976461 rc 0/0 00000100:00100000:2.0:1783520936.229636:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1783520936.229641:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44960 00000100:00100000:2.0:1783520936.229648:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351313b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993279360:0@lo:6:osp-syn-1-0.0 00000004:00080000:3.0:1783520936.229652:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513bc0700/2, rc 0, transno 4294976461 00000100:00100000:3.0:1783520936.229666:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993277696:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.229671:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0269ff lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x890:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.229680:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0269ff lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x890:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.229685:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.229695:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0269ff lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x890:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.229702:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0269ff lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x890:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.229715:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993279360 00010000:00010000:0.0:1783520936.229717:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.229721:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993279360 00010000:00010000:3.0:1783520936.229731:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.229738:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0269ff lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x890:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.229748:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbebe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.229752:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0269ff lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x890:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.229759:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350544d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993279488:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.229765:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0269ff lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x890:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.229800:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.229806:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513bc0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+893:3626:x1870152993277568:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3288us (3535us total) trans 4294976462 rc 0/0 00000100:00100000:0.0:1783520936.229816:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44959 00010000:00010000:3.0:1783520936.229825:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0269ff lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x890:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.229835:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993279488 00010000:00010000:3.0:1783520936.229836:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc0269ff lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x890:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.229836:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363d3a5180/2, rc 0, transno 4294976462 00000100:00100000:0.0:1783520936.229842:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993279488 00000100:00100000:1.0:1783520936.229844:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44966 00000100:00100000:2.0:1783520936.229851:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993277568:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.229861:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502250e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+893:3626:x1870152993279360:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.229869:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.229874:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.229877:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00002000:00080000:1.0:1783520936.229877:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2199 count 1 00000100:00100000:0.0:1783520936.229882:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44967 00000100:00100000:0.0:1783520936.229894:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350544e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+893:3626:x1870152993279488:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.229895:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026a30 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x897:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.229909:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000206e029f/0xd5401cbabc022595 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x897:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a08 expref: 113 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.229912:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2200 count 1 00010000:00010000:1.0:1783520936.229915:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x897:0x0].0x0 since=38s v=(0 0) 00000100:00100000:3.0:1783520936.229918:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351635b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993279616:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.229920:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x897:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.229930:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026a30 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x897:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.229931:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026a0d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x892:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.229931:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026a37 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x898:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.229959:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.229959:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000206e029f/0xd5401cbabc022595 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x897:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a08 expref: 113 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.229966:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003d830c39/0xd5401cbabc02259c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x898:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a16 expref: 113 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.229978:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026a0d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x892:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.229989:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000206e029f/0xd5401cbabc022595 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x897:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a08 expref: 113 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.229991:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x898:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.230002:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x898:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.230003:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026a0d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x892:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.230013:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026a37 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x898:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.230016:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026a0d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x892:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.230017:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514dd6680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993279744:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.230044:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.230052:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.230057:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003d830c39/0xd5401cbabc02259c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x898:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a16 expref: 113 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.230058:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026a0d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x892:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.230062:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993279616 00000100:00100000:1.0:1783520936.230062:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470000 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.230069:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993279616 00010000:00010000:2.0:1783520936.230081:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.230085:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003d830c39/0xd5401cbabc02259c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x898:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a16 expref: 113 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.230088:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026a0d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x892:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.230107:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44968 00010000:00010000:2.0:1783520936.230113:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026a0d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x892:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.230113:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515476a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993279872:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.230122:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516312a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+894:3627:x1870152993279616:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.230138:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026a0d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x892:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.230141:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2201 count 1 00000100:00100000:0.0:1783520936.230148:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80900 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.230153:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026a0d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x892:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.230157:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026a3e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x899:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.230180:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009811f2c9/0xd5401cbabc0225aa lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x899:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a24 expref: 113 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.230186:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.230191:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.230194:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.230197:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x899:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.230201:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x899:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.230205:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026a3e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x899:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.230230:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009811f2c9/0xd5401cbabc0225aa lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x899:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a24 expref: 113 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.230243:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009811f2c9/0xd5401cbabc0225aa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x899:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a24 expref: 113 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.230255:0:35282:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d45249a2/0xd5401cbabc022579 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x893:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29d0 expref: 113 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.230257:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516312680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993280000:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.230276:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515476680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993278464:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.230286:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351313a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+895:3628:x1870152993278080:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3267us (3321us total) trans 4294976463 rc 0/0 00010000:00010000:0.0:1783520936.230292:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026a14 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x893:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.230297:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44961 00000100:00100000:3.0:1783520936.230303:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e780 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.230308:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026a14 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x893:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.230317:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026a14 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x893:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.230326:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026a14 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x893:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.230334:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026a14 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x893:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.230337:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515477b80/2, rc 0, transno 4294976463 00000100:00100000:2.0:1783520936.230354:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515477b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993278080:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.230356:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.230399:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.230406:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026a14 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x893:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.230414:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.230419:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026a14 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x893:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.230426:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516311c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993280128:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.230431:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026a14 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x893:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.230446:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026a14 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x893:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.230455:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026a14 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x893:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.230456:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.230479:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.230482:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.230485:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.230500:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515474700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+895:3625:x1870152993278336:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2798us (2935us total) trans 4294976464 rc 0/0 00000100:00100000:0.0:1783520936.230508:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44962 00000004:00080000:2.0:1783520936.230526:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515475500/2, rc 0, transno 4294976464 00000100:00100000:0.0:1783520936.230527:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993280128 00000100:00100000:0.0:1783520936.230533:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993280128 00000100:00100000:2.0:1783520936.230542:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515475500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993278336:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.230565:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44969 00000100:00100000:0.0:1783520936.230575:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503370380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+895:3628:x1870152993280128:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.230588:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2202 count 1 00000100:00100000:2.0:1783520936.230592:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:0.0:1783520936.230601:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026a45 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x89a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.230602:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503371880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993280256:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.230620:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000265a8fc1/0xd5401cbabc0225b1 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x89a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a32 expref: 112 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.230628:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x89a:0x0].0x0 since=38s v=(0 0) 00000100:00100000:2.0:1783520936.230635:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.230635:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x89a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.230643:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026a45 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x89a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.230665:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000265a8fc1/0xd5401cbabc0225b1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x89a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a32 expref: 112 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.230666:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993280256 00000100:00100000:3.0:1783520936.230673:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993280256 00010000:00010000:0.0:1783520936.230679:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000265a8fc1/0xd5401cbabc0225b1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x89a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a32 expref: 112 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.230692:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515476680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993280384:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.230707:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44970 00000100:00100000:3.0:1783520936.230717:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+896:3626:x1870152993280256:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520936.230731:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2203 count 1 00000100:00100000:0.0:1783520936.230736:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80cc0 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.230745:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026a4c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x89b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.230759:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002e058a76/0xd5401cbabc0225b8 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x89b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a40 expref: 112 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.230772:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x89b:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.230775:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x89b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.230779:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026a4c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x89b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.230799:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002e058a76/0xd5401cbabc0225b8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x89b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a40 expref: 112 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.230809:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002e058a76/0xd5401cbabc0225b8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x89b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a40 expref: 112 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.230819:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516311880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993280512:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.230891:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e840 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.231305:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363d3a6d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993278976:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.231322:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516310000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993279104:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.231324:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026a22 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x896:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.231346:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514dd7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993279232:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.231347:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026a1b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x895:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.231356:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.231422:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.231447:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.231467:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a29 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x894:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.231511:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002481920 00000100:00100000:2.0:1783520936.231517:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002481920 00000100:00100000:0.0:1783520936.231537:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515476a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993279872:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.231545:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002540c41a/0xd5401cbabc022587 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x895:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29ec expref: 113 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.231560:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026a37 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x898:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.231564:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002481792 00000100:00100000:1.0:1783520936.231571:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002481792 00000100:00100000:3.0:1783520936.231574:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516312680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993280000:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.231581:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002482048 00000100:00100000:2.0:1783520936.231586:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002482048 00010000:00010000:3.0:1783520936.231594:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026a3e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x899:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.231594:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e719db35/0xd5401cbabc022ae3 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x896:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31d3 expref: 115 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.231604:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002afd2964/0xd5401cbabc022580 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x894:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29de expref: 115 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.231641:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36554 00000100:00100000:1.0:1783520936.231652:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514dd6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+115:2387:x1870153002481920:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520936.231659:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36555 00010000:00010000:1.0:1783520936.231662:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.231672:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514dd7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+115:2390:x1870153002481792:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.231673:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002540c41a/0xd5401cbabc022587 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x895:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29ec expref: 115 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.231675:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36556 00010000:00010000:3.0:1783520936.231688:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.231694:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002540c41a/0xd5401cbabc022587 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x895:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29ec expref: 115 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.231695:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575ca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+115:2390:x1870153002482048:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.231698:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e719db35/0xd5401cbabc022ae3 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x896:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31d3 expref: 115 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.231712:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.231722:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002afd2964/0xd5401cbabc022580 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x894:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29de expref: 115 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.231729:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e719db35/0xd5401cbabc022ae3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x896:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31d3 expref: 115 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.231733:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002540c41a/0xd5401cbabc022587 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x895:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29ec expref: 115 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.231741:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002afd2964/0xd5401cbabc022580 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x894:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29de expref: 115 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.231764:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e719db35/0xd5401cbabc022ae3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x896:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 46137344->46141439) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f31d3 expref: 114 pid: 35282 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.231765:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x895:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.231772:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002afd2964/0xd5401cbabc022580 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x894:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f29de expref: 114 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.231775:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x895:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.231789:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026a1b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x895:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.231792:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x896:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.231794:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x894:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.231802:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x896:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.231805:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x894:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.231812:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026a1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x895:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.231813:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026a22 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x896:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.231814:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a29 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x894:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.231841:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026a22 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x896:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.231842:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a29 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x894:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.231847:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.231877:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.231882:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.231884:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514dd6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+112:2387:x1870153002481920:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 233us (533us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.231894:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36554 00000100:00100000:3.0:1783520936.231906:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514dd7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+111:2390:x1870153002481792:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 237us (483us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.231914:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575ca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+111:2390:x1870153002482048:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 222us (465us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.231921:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36555 00000100:00100000:2.0:1783520936.231929:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36556 00000100:00100000:1.0:1783520936.231938:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514dd6680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993279744:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.231952:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026a30 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x897:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.231963:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026a22 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x896:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.231973:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026a22 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x896:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.231982:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026a22 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x896:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.231984:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a29 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x894:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.231997:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026a22 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x896:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.232019:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a29 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x894:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.232029:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.232030:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a29 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x894:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.232039:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.232042:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a29 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x894:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.232044:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026a22 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x896:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.232064:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.232067:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026a22 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x896:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.232072:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.232074:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026a22 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x896:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.232085:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026a22 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x896:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.232090:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026a22 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x896:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.232104:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.232107:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a29 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x894:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.232113:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.232116:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a29 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x894:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.232121:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a29 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x894:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.232130:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.232132:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a29 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x894:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.232140:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a29 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x894:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.232149:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350d166a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+898:3628:x1870152993278848:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3354us (3436us total) trans 4294976465 rc 0/0 00000100:00100000:2.0:1783520936.232158:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44965 00010000:00010000:1.0:1783520936.232158:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.232164:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.232167:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.232176:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350794df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+897:3625:x1870152993278592:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3619us (3780us total) trans 4294976466 rc 0/0 00000100:00100000:0.0:1783520936.232184:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44963 00000004:00080000:2.0:1783520936.232197:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351313b100/2, rc 0, transno 4294976465 00000100:00100000:2.0:1783520936.232208:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351313b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993278848:0@lo:6:osp-syn-1-0.0 00000004:00080000:3.0:1783520936.232240:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350794ca80/2, rc 0, transno 4294976466 00000100:00100000:3.0:1783520936.232252:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350794ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993278592:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.232255:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503371f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993280640:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.232290:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.232305:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515476680 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993280384:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.232321:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026a45 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x89a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.232331:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.232393:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514d28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993280768:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.232396:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516311880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993280512:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.232396:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.232416:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026a4c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x89b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.232417:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002482432 00000100:00100000:1.0:1783520936.232424:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002482432 00000100:00100000:2.0:1783520936.232434:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.232439:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002482176 00000100:00100000:0.0:1783520936.232444:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002482176 00010000:00010000:1.0:1783520936.232449:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009811f2c9/0xd5401cbabc0225aa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x899:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a24 expref: 110 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.232453:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993280640 00000100:00100000:3.0:1783520936.232458:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993280640 00010000:00010000:0.0:1783520936.232465:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000206e029f/0xd5401cbabc022595 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x897:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a08 expref: 111 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.232483:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993280768 00000100:00100000:2.0:1783520936.232502:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993280768 00000100:00100000:3.0:1783520936.232508:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44971 00000100:00100000:1.0:1783520936.232514:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36557 00000100:00100000:3.0:1783520936.232519:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363d3a6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+898:3626:x1870152993280640:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.232525:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36558 00000100:00100000:1.0:1783520936.232530:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515476a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+111:2389:x1870153002482432:12345-192.168.204.33@tcp:103:ldlm_bl.0 00002000:00080000:3.0:1783520936.232535:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2204 count 1 00010000:00010000:1.0:1783520936.232541:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.232542:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44972 00000100:00100000:0.0:1783520936.232542:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515474380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+111:2389:x1870153002482176:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.232553:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009811f2c9/0xd5401cbabc0225aa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x899:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a24 expref: 111 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.232554:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc026a53 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x89c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.232556:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.232557:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+898:3627:x1870152993280768:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.232567:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000206e029f/0xd5401cbabc022595 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x897:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a08 expref: 111 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.232581:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2205 count 1 00010000:00010000:3.0:1783520936.232587:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003cf451f7/0xd5401cbabc0225bf lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x89c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a4e expref: 111 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.232589:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009811f2c9/0xd5401cbabc0225aa lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x899:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a24 expref: 111 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.232598:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026a5a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x89d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.232610:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000206e029f/0xd5401cbabc022595 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x897:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a08 expref: 111 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.232615:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x89c:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.232625:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x89c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.232628:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009811f2c9/0xd5401cbabc0225aa lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x899:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a24 expref: 111 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.232633:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e72da8af/0xd5401cbabc0225c6 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x89d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a5c expref: 111 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.232635:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc026a53 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x89c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.232645:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000206e029f/0xd5401cbabc022595 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x897:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a08 expref: 111 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.232660:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x899:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.232664:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x89d:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.232669:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x899:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.232672:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x89d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.232675:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x897:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.232677:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003cf451f7/0xd5401cbabc0225bf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x89c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a4e expref: 109 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.232678:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026a3e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x899:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.232681:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026a5a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x89d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.232683:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x897:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.232694:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026a30 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x897:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.232707:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003cf451f7/0xd5401cbabc0225bf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x89c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a4e expref: 109 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.232708:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026a3e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x899:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.232719:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026a30 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x897:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.232729:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e72da8af/0xd5401cbabc0225c6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x89d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a5c expref: 109 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.232745:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516311880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993280896:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.232752:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e72da8af/0xd5401cbabc0225c6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x89d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a5c expref: 109 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.232754:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.232758:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.232768:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9d500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993281024:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.232786:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515476a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+109:2389:x1870153002482432:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 260us (456us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.232790:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e840 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.232791:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515474380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+109:2389:x1870153002482176:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 254us (399us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.232804:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1000 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.232804:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36558 00010000:00010000:3.0:1783520936.232808:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026a3e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x899:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.232813:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36557 00010000:00010000:3.0:1783520936.232818:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026a3e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x899:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.232826:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026a3e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x899:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.232831:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026a3e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x899:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.232844:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.232849:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.232857:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.232864:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026a3e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x899:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.232874:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.232879:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026a3e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x899:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.232880:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.232891:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026a3e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x899:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.232903:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002482304 00010000:00010000:2.0:1783520936.232907:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026a3e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x899:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.232907:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002482304 00010000:00010000:2.0:1783520936.232918:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026a3e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x899:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.232924:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003d830c39/0xd5401cbabc02259c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x898:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a16 expref: 108 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.232928:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516312a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+899:3627:x1870152993279616:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2810us (2972us total) trans 4294976467 rc 0/0 00010000:00010000:2.0:1783520936.232938:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.232942:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44968 00010000:00010000:2.0:1783520936.232943:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.232949:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.232969:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002482560 00000100:00100000:0.0:1783520936.232971:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002482560 00000004:00080000:2.0:1783520936.232981:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351635b480/2, rc 0, transno 4294976467 00010000:00010000:0.0:1783520936.232984:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000265a8fc1/0xd5401cbabc0225b1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x89a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a32 expref: 109 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.232999:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351635b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993279616:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.233005:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36559 00000100:00100000:0.0:1783520936.233013:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515476680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+109:2388:x1870153002482304:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.233019:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.233024:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.233025:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003d830c39/0xd5401cbabc02259c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x898:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a16 expref: 109 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.233032:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36560 00010000:00010000:0.0:1783520936.233041:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003d830c39/0xd5401cbabc02259c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x898:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a16 expref: 109 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.233049:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f2450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+109:2388:x1870153002482560:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.233052:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026a1b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x895:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.233057:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003d830c39/0xd5401cbabc02259c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x898:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a16 expref: 109 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.233061:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.233073:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000265a8fc1/0xd5401cbabc0225b1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x89a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a32 expref: 109 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.233073:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026a1b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x895:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.233077:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x898:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.233086:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x898:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.233096:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026a37 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x898:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.233104:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026a1b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x895:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.233106:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000265a8fc1/0xd5401cbabc0225b1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x89a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a32 expref: 108 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.233110:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026a37 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x898:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.233130:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026a1b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x895:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.233142:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000265a8fc1/0xd5401cbabc0225b1 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x89a:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a32 expref: 108 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.233161:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x89a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.233162:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.233163:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.233166:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x89a:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.233171:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026a45 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.233175:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.233183:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026a1b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x895:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.233184:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026a45 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x89a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.233190:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515476680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+107:2388:x1870153002482304:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 177us (342us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.233208:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4e00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520936.233209:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36559 00010000:00010000:1.0:1783520936.233216:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026a1b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x895:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.233224:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.233227:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026a1b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x895:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.233228:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002482688 00000100:00100000:0.0:1783520936.233233:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002482688 00010000:00010000:1.0:1783520936.233248:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026a1b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x895:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.233250:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f2450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+107:2388:x1870153002482560:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 206us (369us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.233253:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002e058a76/0xd5401cbabc0225b8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x89b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a40 expref: 107 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.233271:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36560 00010000:00010000:1.0:1783520936.233283:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026a1b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x895:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.233298:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36561 00010000:00010000:1.0:1783520936.233306:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.233309:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575e300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+106:2390:x1870153002482688:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.233310:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.233315:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.233320:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.233328:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002e058a76/0xd5401cbabc0225b8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x89b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a40 expref: 106 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.233329:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:1.0:1783520936.233335:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026a45 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.233342:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516312680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993281152:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.233348:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002e058a76/0xd5401cbabc0225b8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x89b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a40 expref: 106 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.233354:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026a45 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.233414:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.233422:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026a45 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.233426:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002e058a76/0xd5401cbabc0225b8 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x89b:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a40 expref: 106 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.233440:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026a45 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x89a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.233446:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x89b:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.233451:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x89b:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.233456:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993281152 00010000:00010000:0.0:1783520936.233456:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026a4c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.233457:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.233462:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993281152 00010000:00010000:3.0:1783520936.233469:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.233472:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026a4c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x89b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.233477:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026a45 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x89a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.233498:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.233503:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026a45 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x89a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.233510:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44973 00010000:00010000:0.0:1783520936.233511:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.233519:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026a45 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x89a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.233526:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517244e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+900:3627:x1870152993281152:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.233539:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026a45 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x89a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.233541:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575e300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+105:2390:x1870153002482688:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 233us (519us total) trans 0 rc 0/0 00002000:00080000:2.0:1783520936.233545:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2206 count 1 00010000:00010000:3.0:1783520936.233560:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026a45 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x89a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.233563:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36561 00000100:00100000:1.0:1783520936.233564:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503370380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+900:3628:x1870152993280128:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2987us (3103us total) trans 4294976468 rc 0/0 00010000:00010000:2.0:1783520936.233565:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026a61 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x89e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.233586:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44969 00010000:00010000:2.0:1783520936.233595:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000024c97d7/0xd5401cbabc0229d2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x89e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f30c2 expref: 104 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.233597:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026a4c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.233599:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.233603:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.233607:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.233615:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x89e:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.233617:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026a4c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.233621:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x89e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.233631:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026a61 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x89e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.233635:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026a4c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.233649:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026a4c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x89b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.233661:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000024c97d7/0xd5401cbabc0229d2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x89e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30c2 expref: 104 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.233672:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.233675:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000024c97d7/0xd5401cbabc0229d2 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x89e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30c2 expref: 104 pid: 8408 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.233681:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513139c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+900:3628:x1870152993278720:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4906us (5071us total) trans 4294976469 rc 0/0 00000100:00100000:1.0:1783520936.233696:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44964 00000100:00100000:2.0:1783520936.233699:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517247100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993281280:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.233717:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.233723:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026a4c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x89b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.233731:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.233735:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026a4c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x89b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.233738:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1780 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.233747:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026a4c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x89b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.233762:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026a4c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x89b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.233763:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516311c00/2, rc 0, transno 4294976468 00000100:00100000:0.0:1783520936.233772:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+900:3626:x1870152993280256:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3054us (3136us total) trans 4294976470 rc 0/0 00010000:00010000:1.0:1783520936.233775:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026a4c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x89b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.233779:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516311c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993280128:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.233785:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44970 00000004:00080000:3.0:1783520936.233789:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503371880/2, rc 0, transno 4294976470 00000100:00100000:3.0:1783520936.233801:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503371880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993280256:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.233812:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:0.0:1783520936.233815:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026a37 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x898:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.233816:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.233820:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.233824:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514fdc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993281408:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.233826:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.233837:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026a37 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x898:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.233838:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:0.0:1783520936.233847:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026a37 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x898:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.233852:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435038eea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993281536:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.233854:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026a37 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x898:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.233874:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520936.233928:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.233929:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.233935:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026a37 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x898:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.233944:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.233948:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026a37 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x898:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.233957:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026a37 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x898:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.233964:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350544e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+900:3626:x1870152993279488:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4070us (4162us total) trans 4294976471 rc 0/0 00000100:00100000:0.0:1783520936.233973:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44967 00010000:00010000:2.0:1783520936.233974:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026a37 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x898:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.233984:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000512f674c/0xd5401cbabc026a37 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x898:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.233990:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993281408 00000100:00100000:0.0:1783520936.233995:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993281408 00010000:00010000:2.0:1783520936.234002:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.234006:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.234009:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520936.234020:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350544d180/2, rc 0, transno 4294976471 00000100:00100000:0.0:1783520936.234030:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44974 00000100:00100000:3.0:1783520936.234035:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350544d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993279488:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.234041:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516310000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+900:3626:x1870152993281408:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.234046:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00080000:0.0:1783520936.234057:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2207 count 1 00000100:00100000:3.0:1783520936.234065:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993281536 00000100:00100000:3.0:1783520936.234070:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993281536 00010000:00010000:0.0:1783520936.234071:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a68 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x89f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.234081:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.234094:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000232a493b/0xd5401cbabc0225d4 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x89f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a78 expref: 104 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.234103:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x89f:0x0].0x0 since=38s v=(0 0) 00000100:00100000:3.0:1783520936.234109:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44975 00010000:00010000:0.0:1783520936.234111:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x89f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.234118:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a68 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x89f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.234120:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435038ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993281664:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.234124:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435038ed180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+901:3625:x1870152993281536:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.234133:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002482944 00000100:00100000:1.0:1783520936.234138:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002482944 00002000:00080000:3.0:1783520936.234146:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2208 count 1 00010000:00010000:1.0:1783520936.234159:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e72da8af/0xd5401cbabc0225c6 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x89d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a5c expref: 105 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.234160:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000232a493b/0xd5401cbabc0225d4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x89f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a78 expref: 105 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.234161:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.234162:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026a6f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8a0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.234188:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000232a493b/0xd5401cbabc0225d4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x89f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a78 expref: 105 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.234199:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36562 00000100:00100000:0.0:1783520936.234204:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515474380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993281792:192.168.204.33@tcp:104: 00000004:00080000:2.0:1783520936.234208:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513138000/2, rc 0, transno 4294976469 00000100:00100000:1.0:1783520936.234214:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+105:2387:x1870153002482944:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.234222:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513138000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993278720:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.234226:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.234234:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e72da8af/0xd5401cbabc0225c6 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x89d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a5c expref: 105 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.234244:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80840 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.234246:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e72da8af/0xd5401cbabc0225c6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x89d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a5c expref: 105 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.234265:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x89d:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.234269:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:1.0:1783520936.234269:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x89d:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.234276:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026a5a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.234277:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993281920:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.234292:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026a5a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x89d:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.234294:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026a30 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x897:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.234314:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.234317:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026a30 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x897:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.234319:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.234330:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026a30 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x897:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.234338:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026a30 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x897:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.234355:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+105:2387:x1870153002482944:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 145us (277us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.234380:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.234393:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36562 00000100:00100000:2.0:1783520936.234398:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993281664 00000100:00100000:2.0:1783520936.234403:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993281664 00010000:00010000:1.0:1783520936.234413:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.234419:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026a30 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x897:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.234427:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb200 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520936.234432:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993281920 00010000:00010000:1.0:1783520936.234432:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026a30 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x897:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.234438:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993281920 00010000:00010000:3.0:1783520936.234456:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007daca1d3/0xd5401cbabc0225db lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a86 expref: 104 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.234459:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026a30 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x897:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.234462:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44976 00000100:00100000:0.0:1783520936.234470:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502250e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+904:3626:x1870152993279360:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4612us (4784us total) trans 4294976472 rc 0/0 00000100:00100000:2.0:1783520936.234478:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ae5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+904:3627:x1870152993281664:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.234501:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44966 00010000:00010000:1.0:1783520936.234504:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026a30 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x897:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.234510:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2209 count 1 00010000:00010000:1.0:1783520936.234515:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026a30 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x897:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.234524:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44977 00010000:00010000:2.0:1783520936.234525:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026a76 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8a1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.234538:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.234539:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000012566e20/0xd5401cbabc0225e2 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a94 expref: 104 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.234540:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943508ee4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+903:3627:x1870152993281920:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.234543:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.234547:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.234553:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a1:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.234556:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.234558:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026a76 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.234560:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2210 count 1 00010000:00010000:0.0:1783520936.234574:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026a7d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8a2:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.234581:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000012566e20/0xd5401cbabc0225e2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a94 expref: 104 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.234586:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000328c5aa5/0xd5401cbabc0225e9 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2aa2 expref: 104 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.234591:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.234606:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a0:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.234607:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a2:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.234611:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000012566e20/0xd5401cbabc0225e2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a94 expref: 104 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.234616:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.234617:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.234625:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026a7d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.234628:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026a6f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.234641:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e472300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993282048:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.234661:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002482816 00000100:00100000:1.0:1783520936.234667:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002482816 00010000:00010000:0.0:1783520936.234669:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000328c5aa5/0xd5401cbabc0225e9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2aa2 expref: 104 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.234680:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351313b480/2, rc 0, transno 4294976472 00010000:00010000:0.0:1783520936.234685:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000328c5aa5/0xd5401cbabc0225e9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2aa2 expref: 105 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.234690:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003cf451f7/0xd5401cbabc0225bf lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x89c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a4e expref: 105 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.234708:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515476a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993282176:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.234727:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36563 00000100:00100000:2.0:1783520936.234736:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e471f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993282304:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.234739:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351313b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993279360:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.234740:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575ca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+105:2387:x1870153002482816:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.234748:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80a80 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.234755:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.234762:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003cf451f7/0xd5401cbabc0225bf lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x89c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a4e expref: 105 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.234774:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003cf451f7/0xd5401cbabc0225bf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x89c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a4e expref: 105 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.234782:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.234792:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007daca1d3/0xd5401cbabc0225db lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a86 expref: 105 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.234797:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x89c:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.234808:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x89c:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.234811:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993282304 00010000:00010000:3.0:1783520936.234813:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007daca1d3/0xd5401cbabc0225db lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a86 expref: 105 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.234815:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc026a53 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.234816:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993282304 00000100:00100000:2.0:1783520936.234835:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44978 00010000:00010000:1.0:1783520936.234840:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc026a53 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x89c:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.234841:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516310700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993282432:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.234844:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350626e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+904:3625:x1870152993282304:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.234863:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2211 count 1 00010000:00010000:1.0:1783520936.234866:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.234880:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026a84 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8a3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.234894:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000081309932/0xd5401cbabc0225f0 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ab0 expref: 105 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.234896:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575ca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+105:2387:x1870153002482816:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 159us (310us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.234912:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a3:0x0].0x0 since=38s v=(0 0) 00000100:00100000:1.0:1783520936.234914:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36563 00010000:00010000:2.0:1783520936.234918:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.234924:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026a84 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.234946:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000081309932/0xd5401cbabc0225f0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ab0 expref: 104 pid: 8409 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.234955:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000081309932/0xd5401cbabc0225f0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ab0 expref: 104 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.234965:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350626d180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993282560:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.235001:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1900 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.235014:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.235017:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1180 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.235048:0:35276:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e72da8af/0xd5401cbabc0225c6 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x89d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a5c expref: 104 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.235067:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9d500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993281024:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.235081:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026a5a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.235090:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026a5a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.235097:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026a5a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.235104:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026a5a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.235111:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026a5a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x89d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.235117:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003cf451f7/0xd5401cbabc0225bf lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x89c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a4e expref: 103 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.235135:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516311880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993280896:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.235135:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.235148:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.235155:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026a5a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x89d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.235156:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc026a53 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.235171:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.235175:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026a5a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x89d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.235185:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026a5a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x89d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.235198:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026a5a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x89d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.235206:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026a5a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x89d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.235222:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.235225:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.235227:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.235252:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+905:3627:x1870152993280768:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2698us (2819us total) trans 4294976473 rc 0/0 00000100:00100000:2.0:1783520936.235261:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44972 00010000:00010000:3.0:1783520936.235267:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc026a53 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.235274:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc026a53 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.235280:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc026a53 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.235284:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514d28a80/2, rc 0, transno 4294976473 00010000:00010000:3.0:1783520936.235285:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc026a53 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x89c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.235300:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514d28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993280768:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.235303:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.235318:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.235324:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc026a53 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x89c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.235325:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515474380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993281792:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.235336:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:1.0:1783520936.235339:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.235344:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a68 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x89f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.235345:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc026a53 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x89c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.235346:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350626d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993282688:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.235391:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc026a53 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x89c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.235404:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363d3a6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+905:3626:x1870152993280640:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2887us (3113us total) trans 4294976474 rc 0/0 00000100:00100000:3.0:1783520936.235410:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44971 00010000:00010000:1.0:1783520936.235412:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc026a53 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x89c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.235421:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007a4fc52d/0xd5401cbabc026a53 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x89c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.235431:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.235437:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.235439:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.235442:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.235455:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993282688 00000100:00100000:0.0:1783520936.235459:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993282688 00000004:00080000:2.0:1783520936.235480:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503371f80/2, rc 0, transno 4294976474 00000100:00100000:0.0:1783520936.235484:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44979 00000100:00100000:0.0:1783520936.235504:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350626ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+905:3625:x1870152993282688:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.235507:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503371f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993280640:0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.235524:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2212 count 1 00010000:00010000:0.0:1783520936.235539:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026a8b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8a4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.235553:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ba25b7b0/0xd5401cbabc0225f7 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2abe expref: 102 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.235559:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516311880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993282816:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.235563:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a4:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.235568:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.235573:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026a8b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.235592:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.235597:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ba25b7b0/0xd5401cbabc0225f7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2abe expref: 102 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.235609:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ba25b7b0/0xd5401cbabc0225f7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2abe expref: 102 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.235619:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515474380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993282944:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.235628:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993282816 00000100:00100000:2.0:1783520936.235633:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993282816 00000100:00100000:0.0:1783520936.235656:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80840 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.235671:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44980 00000100:00100000:2.0:1783520936.235681:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436342b8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+906:3627:x1870152993282816:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.235694:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2213 count 1 00010000:00010000:2.0:1783520936.235709:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026a92 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8a5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.235722:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b31f1b7c/0xd5401cbabc0225fe lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2acc expref: 102 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.235729:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a5:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.235732:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.235735:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026a92 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.235753:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b31f1b7c/0xd5401cbabc0225fe lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2acc expref: 102 pid: 35275 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.235762:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b31f1b7c/0xd5401cbabc0225fe lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2acc expref: 102 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.235772:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350626dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993283072:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.235802:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1a80 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.235964:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e472300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993282048:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.235980:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026a76 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.236150:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517247100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993281280:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.236163:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026a61 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x89e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.236188:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516310700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993282432:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.236265:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026a6f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.236399:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350626d180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993282560:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.236413:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026a84 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.236713:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.236740:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002483200 00000100:00100000:3.0:1783520936.236746:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002483200 00000100:00100000:0.0:1783520936.236762:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.236769:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000012566e20/0xd5401cbabc0225e2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a94 expref: 103 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.236800:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002483456 00000100:00100000:2.0:1783520936.236805:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002483456 00000100:00100000:3.0:1783520936.236808:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36564 00000100:00100000:3.0:1783520936.236819:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515476680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+104:2387:x1870153002483200:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.236827:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000328c5aa5/0xd5401cbabc0225e9 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2aa2 expref: 104 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.236830:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.236839:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000012566e20/0xd5401cbabc0225e2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8a1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a94 expref: 104 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.236851:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000012566e20/0xd5401cbabc0225e2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8a1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a94 expref: 104 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.236862:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36565 00010000:00010000:3.0:1783520936.236872:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000012566e20/0xd5401cbabc0225e2 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8a1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a94 expref: 104 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.236872:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514dd6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+104:2387:x1870153002483456:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.236883:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.236888:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000328c5aa5/0xd5401cbabc0225e9 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8a2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2aa2 expref: 103 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.236889:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.236898:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.236902:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000328c5aa5/0xd5401cbabc0225e9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8a2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2aa2 expref: 103 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.236906:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026a76 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.236910:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515476a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993282176:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.236920:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000328c5aa5/0xd5401cbabc0225e9 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8a2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2aa2 expref: 103 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.236927:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026a76 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8a1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.236931:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026a7d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.236932:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.236939:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.236947:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026a7d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.236954:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026a7d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8a2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.236957:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.236967:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.236972:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026a7d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.236981:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026a7d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.236991:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026a7d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.236997:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515476680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+102:2387:x1870153002483200:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 177us (281us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.236999:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514dd6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+102:2387:x1870153002483456:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 126us (233us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.237012:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026a7d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.237020:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36565 00000100:00100000:3.0:1783520936.237032:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36564 00010000:00010000:1.0:1783520936.237044:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.237050:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.237056:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026a7d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.237063:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.237067:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026a7d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.237073:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026a7d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.237081:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026a7d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.237087:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000022e5f82b/0xd5401cbabc026a7d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8a2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.237108:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.237110:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.237112:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.237124:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026a76 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.237130:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026a76 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.237136:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026a76 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.237140:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026a76 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.237157:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.237159:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943508ee4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+907:3627:x1870152993281920:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2622us (2846us total) trans 4294976475 rc 0/0 00000100:00100000:1.0:1783520936.237170:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44977 00000004:00080000:0.0:1783520936.237182:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943508ee4000/2, rc 0, transno 4294976475 00010000:00010000:1.0:1783520936.237192:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520936.237197:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993281920:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.237200:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026a76 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.237211:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.237215:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026a76 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.237224:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026a76 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.237237:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026a76 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.237245:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026a76 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8a1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.237256:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.237272:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ae5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+907:3627:x1870152993281664:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2796us (3108us total) trans 4294976476 rc 0/0 00010000:00010000:1.0:1783520936.237272:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.237276:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.237283:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.237285:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44976 00000004:00080000:0.0:1783520936.237290:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435038ec380/2, rc 0, transno 4294976476 00000100:00100000:1.0:1783520936.237299:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002483072 00000100:00100000:1.0:1783520936.237305:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002483072 00000100:00100000:0.0:1783520936.237309:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435038ec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993281664:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.237316:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516310700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993283200:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.237326:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000232a493b/0xd5401cbabc0225d4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x89f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a78 expref: 101 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.237355:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36566 00000100:00100000:0.0:1783520936.237381:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.237387:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.237393:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514dd7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+101:2387:x1870153002483072:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.237403:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.237408:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000232a493b/0xd5401cbabc0225d4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x89f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a78 expref: 101 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.237419:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000232a493b/0xd5401cbabc0225d4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x89f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a78 expref: 101 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.237436:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000232a493b/0xd5401cbabc0225d4 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x89f:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a78 expref: 101 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.237437:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.237453:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x89f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.237457:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x89f:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.237462:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a68 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.237464:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517247100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993283328:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.237470:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.237481:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a68 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x89f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.237520:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.237548:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514dd7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+100:2387:x1870153002483072:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 160us (298us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.237557:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36566 00000100:00100000:3.0:1783520936.237567:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002483712 00000100:00100000:3.0:1783520936.237574:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002483712 00000100:00100000:2.0:1783520936.237575:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.237576:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002483328 00000100:00100000:1.0:1783520936.237582:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002483328 00010000:00010000:1.0:1783520936.237598:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000024c97d7/0xd5401cbabc0229d2 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x89e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30c2 expref: 100 pid: 8408 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.237611:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.237623:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002483584 00000100:00100000:1.0:1783520936.237629:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002483584 00000100:00100000:2.0:1783520936.237640:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002483840 00000100:00100000:2.0:1783520936.237647:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002483840 00010000:00010000:1.0:1783520936.237649:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000081309932/0xd5401cbabc0225f0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ab0 expref: 101 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.237649:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.237672:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002483968 00010000:00010000:2.0:1783520936.237674:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ba25b7b0/0xd5401cbabc0225f7 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2abe expref: 103 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.237678:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002483968 00000100:00100000:1.0:1783520936.237682:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36567 00000100:00100000:1.0:1783520936.237696:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514dd6680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+104:2387:x1870153002483328:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.237705:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b31f1b7c/0xd5401cbabc0225fe lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2acc expref: 104 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.237707:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36568 00010000:00010000:1.0:1783520936.237707:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.237718:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000024c97d7/0xd5401cbabc0229d2 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x89e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30c2 expref: 104 pid: 8408 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.237721:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514dd7100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+104:2387:x1870153002483584:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.237729:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007daca1d3/0xd5401cbabc0225db lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a86 expref: 103 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.237735:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36569 00010000:00010000:2.0:1783520936.237736:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.237746:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000081309932/0xd5401cbabc0225f0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8a3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ab0 expref: 104 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.237749:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000024c97d7/0xd5401cbabc0229d2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x89e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30c2 expref: 104 pid: 8408 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.237752:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530d500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+104:2388:x1870153002483840:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520936.237765:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36570 00010000:00010000:0.0:1783520936.237768:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.237776:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000081309932/0xd5401cbabc0225f0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8a3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ab0 expref: 104 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.237778:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ba25b7b0/0xd5401cbabc0225f7 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8a4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2abe expref: 104 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.237781:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351530dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+104:2388:x1870153002483968:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.237785:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000024c97d7/0xd5401cbabc0229d2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x89e:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 25165824->25169919) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30c2 expref: 104 pid: 8408 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.237797:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.237809:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b31f1b7c/0xd5401cbabc0225fe lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8a5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2acc expref: 104 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.237809:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ba25b7b0/0xd5401cbabc0225f7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8a4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2abe expref: 104 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.237813:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x89e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.237814:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000081309932/0xd5401cbabc0225f0 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8a3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ab0 expref: 103 pid: 8409 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.237822:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x89e:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.237835:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026a61 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.237844:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b31f1b7c/0xd5401cbabc0225fe lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8a5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2acc expref: 103 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.237846:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.237847:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026a61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x89e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.237848:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.237856:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.237859:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.237868:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026a8b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.237871:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026a84 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.237873:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.237880:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.237882:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.237888:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026a8b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8a4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.237893:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026a92 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.237897:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026a84 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8a3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.237905:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514dd6680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+102:2387:x1870153002483328:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 210us (470us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.237921:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026a92 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8a5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.237923:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36567 00000100:00100000:1.0:1783520936.237942:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36571 00010000:00010000:0.0:1783520936.237944:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.237946:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.237948:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.237951:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f3c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+101:2389:x1870153002483712:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.237959:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.237963:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007daca1d3/0xd5401cbabc0225db lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8a0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a86 expref: 101 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.237971:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530d500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+101:2388:x1870153002483840:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 222us (359us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.237976:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007daca1d3/0xd5401cbabc0225db lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8a0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a86 expref: 101 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.237980:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351530dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+101:2388:x1870153002483968:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 202us (332us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.237987:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514dd7100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+101:2387:x1870153002483584:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 267us (517us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.237988:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36569 00000100:00100000:3.0:1783520936.238002:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36570 00010000:00010000:1.0:1783520936.238004:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007daca1d3/0xd5401cbabc0225db lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8a0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2a86 expref: 100 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.238012:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36568 00010000:00010000:1.0:1783520936.238016:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.238020:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.238024:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026a6f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.238033:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026a6f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8a0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.238036:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993283200 00000100:00100000:3.0:1783520936.238042:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993283200 00010000:00010000:2.0:1783520936.238044:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026a84 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.238049:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.238055:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026a84 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.238065:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026a84 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.238070:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993283328 00000100:00100000:1.0:1783520936.238070:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f3c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+97:2389:x1870153002483712:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 121us (717us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.238075:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993283328 00010000:00010000:2.0:1783520936.238085:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026a84 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.238085:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36571 00000100:00100000:3.0:1783520936.238106:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44981 00010000:00010000:2.0:1783520936.238111:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.238116:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:3.0:1783520936.238117:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350626d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+908:3628:x1870152993283200:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.238124:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026a84 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.238133:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185c00 already unused, calling callback (ffffffffc0bf9e90) 00002000:00080000:3.0:1783520936.238136:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2214 count 1 00010000:00010000:0.0:1783520936.238138:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026a84 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.238146:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44982 00010000:00010000:0.0:1783520936.238150:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026a84 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.238156:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e472300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+909:3625:x1870152993283328:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.238169:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026a84 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.238172:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2215 count 1 00010000:00010000:0.0:1783520936.238180:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026a84 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8a3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.238188:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a99 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8a7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.238198:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000bba8852e/0xd5401cbabc02260c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ae8 expref: 96 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.238201:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.238206:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a7:0x0].0x0 since=38s v=(0 0) 00000100:00100000:2.0:1783520936.238207:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350626e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+909:3625:x1870152993282304:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3361us (3423us total) trans 4294976477 rc 0/0 00010000:00010000:0.0:1783520936.238207:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.238210:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.238215:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.238220:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026aa0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8a6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.238222:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44978 00010000:00010000:1.0:1783520936.238222:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a99 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.238229:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a68 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.238249:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000841d2395/0xd5401cbabc0228c1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fb1 expref: 96 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.238251:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a68 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.238257:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e471f80/2, rc 0, transno 4294976477 00010000:00010000:1.0:1783520936.238263:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000bba8852e/0xd5401cbabc02260c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ae8 expref: 96 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.238265:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a68 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.238276:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e471f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993282304:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.238290:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a68 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x89f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.238295:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000bba8852e/0xd5401cbabc02260c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ae8 expref: 96 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.238310:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x8a6:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.238319:0:20415:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b31f1b7c/0xd5401cbabc0225fe lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8a5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2acc expref: 96 pid: 35275 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.238320:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a6:0x0].0x0 since=0s v=(1 0) 00000100:00100000:1.0:1783520936.238326:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9436342b8a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993283456:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.238327:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.238329:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026aa0 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.238342:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:2.0:1783520936.238348:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350626dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993283072:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.238349:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a68 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x89f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.238399:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.238404:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a68 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x89f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.238413:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026a92 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.238415:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a68 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x89f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.238446:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a68 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x89f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.238453:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000584fdda9/0xd5401cbabc026a68 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x89f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.238463:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.238464:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.238465:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.238473:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026a61 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.238478:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026a61 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.238484:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026a61 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x89e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.238501:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026a61 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x89e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.238555:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.238558:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507938a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993283584:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.238561:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516310000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+909:3626:x1870152993281408:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4521us (4632us total) trans 4294976478 rc 0/0 00010000:00010000:2.0:1783520936.238572:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520936.238572:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44974 00010000:00010000:2.0:1783520936.238580:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026a61 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x89e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.238591:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.238595:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026a61 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x89e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.238603:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026a61 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x89e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.238616:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026a61 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x89e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.238618:0:8408:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ba25b7b0/0xd5401cbabc0225f7 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8a4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2abe expref: 95 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.238620:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026a6f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.238642:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026a61 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x89e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.238646:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026a6f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.238650:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515474380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993282944:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.238659:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517244e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+909:3627:x1870152993281152:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5135us (5247us total) trans 4294976479 rc 0/0 00000100:00100000:1.0:1783520936.238672:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44973 00010000:00010000:0.0:1783520936.238673:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026a8b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.238678:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.238679:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026a6f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.238683:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.238688:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.238695:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026a6f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.238710:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516312680/2, rc 0, transno 4294976479 00010000:00010000:3.0:1783520936.238716:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.238727:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520936.238728:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516312680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993281152:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.238733:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026a6f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.238742:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4a00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520936.238744:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.238749:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026a6f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.238750:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026a8b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.238759:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026a6f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.238780:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026a8b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.238781:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514fdc000/2, rc 0, transno 4294976478 00010000:00010000:1.0:1783520936.238781:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026a6f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.238800:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514fdc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993281408:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.238802:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d8e3e60d/0xd5401cbabc026a6f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8a0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.238803:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026a8b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.238825:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507938380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993283840:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.238827:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026a8b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.238832:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.238835:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.238839:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.238852:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993283712:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.238867:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.238869:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470780 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.238885:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993283584 00000100:00100000:3.0:1783520936.238887:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435038ed180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+909:3625:x1870152993281536:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4766us (4842us total) trans 4294976480 rc 0/0 00000100:00100000:1.0:1783520936.238889:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993283584 00000100:00100000:0.0:1783520936.238898:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.238902:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44975 00000100:00100000:3.0:1783520936.238927:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993283840 00000100:00100000:1.0:1783520936.238928:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993283712 00000100:00100000:3.0:1783520936.238931:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993283840 00000004:00080000:2.0:1783520936.238931:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435038eea00/2, rc 0, transno 4294976480 00000100:00100000:1.0:1783520936.238932:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993283712 00000100:00100000:2.0:1783520936.238951:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435038eea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993281536:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.238953:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.238959:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44983 00010000:00010000:0.0:1783520936.238966:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520936.238967:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363573b050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+911:3626:x1870152993283584:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.238973:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026a8b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.238978:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44984 00002000:00080000:1.0:1783520936.238980:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2216 count 1 00010000:00010000:0.0:1783520936.238988:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeac00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.238993:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026a8b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.238995:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026aa7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8a8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.239000:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350793aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993283968:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.239008:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026a8b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.239014:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44985 00010000:00010000:1.0:1783520936.239023:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003ec0d447/0xd5401cbabc022613 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2af6 expref: 94 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.239028:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350626dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+911:3628:x1870152993283840:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.239031:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026a8b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.239036:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.239045:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a8:0x0].0x0 since=38s v=(0 0) 00002000:00080000:3.0:1783520936.239048:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2218 count 1 00010000:00010000:1.0:1783520936.239055:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.239063:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026a8b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8a4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.239066:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc026aae lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8aa:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.239066:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026aa7 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.239073:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c9b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+911:3627:x1870152993283712:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.239084:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.239088:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00002000:00080000:2.0:1783520936.239093:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2217 count 1 00010000:00010000:0.0:1783520936.239093:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.239098:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000cc8b8804/0xd5401cbabc022628 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b12 expref: 94 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.239108:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8aa:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.239109:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026ab5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8a9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.239110:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003ec0d447/0xd5401cbabc022613 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2af6 expref: 94 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.239115:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8aa:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.239125:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc026aae lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.239135:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003ec0d447/0xd5401cbabc022613 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2af6 expref: 94 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.239141:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000025a42e3e/0xd5401cbabc02261a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b04 expref: 94 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.239155:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575e300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993284096:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.239157:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a9:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.239161:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000cc8b8804/0xd5401cbabc022628 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b12 expref: 94 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.239163:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.239174:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026ab5 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.239179:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000cc8b8804/0xd5401cbabc022628 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b12 expref: 94 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.239184:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470f00 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.239189:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350626ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+912:3625:x1870152993282688:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3684us (3756us total) trans 4294976481 rc 0/0 00000100:00100000:0.0:1783520936.239206:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44979 00000100:00100000:1.0:1783520936.239208:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993283968 00000004:00080000:2.0:1783520936.239209:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350626d500/2, rc 0, transno 4294976481 00000100:00100000:1.0:1783520936.239214:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993283968 00000100:00100000:3.0:1783520936.239215:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350793bb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993284224:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.239227:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350626d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993282688:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.239253:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44986 00000100:00100000:1.0:1783520936.239263:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+912:3625:x1870152993283968:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.239271:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507939c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993284352:0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.239277:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2219 count 1 00010000:00010000:1.0:1783520936.239292:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026abc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8ab:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.239298:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.239306:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000126f6394/0xd5401cbabc022621 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b20 expref: 94 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.239312:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.239321:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ab:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.239326:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ab:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.239327:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000841d2395/0xd5401cbabc0228c1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fb1 expref: 94 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.239330:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993284352 00010000:00010000:1.0:1783520936.239332:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026abc lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.239336:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993284352 00010000:00010000:1.0:1783520936.239379:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000126f6394/0xd5401cbabc022621 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b20 expref: 94 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.239395:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000025a42e3e/0xd5401cbabc02261a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b04 expref: 94 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.239395:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000126f6394/0xd5401cbabc022621 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b20 expref: 94 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.239396:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44987 00000100:00100000:0.0:1783520936.239414:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351cc7d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+913:3626:x1870152993284352:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.239443:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000025a42e3e/0xd5401cbabc02261a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b04 expref: 94 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.239444:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000841d2395/0xd5401cbabc0228c1 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fb1 expref: 94 pid: 35276 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.239448:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575d180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993284608:192.168.204.33@tcp:104: 00002000:00080000:0.0:1783520936.239453:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2220 count 1 00010000:00010000:0.0:1783520936.239471:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026ac3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8ac:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.239474:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cc7d880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993284736:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.239502:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b20a2b9a/0xd5401cbabc02262f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b2e expref: 94 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.239507:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.239511:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ac:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.239516:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ac:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.239521:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026ac3 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.239525:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1a80 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.239526:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350793ad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993284480:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.239545:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026a92 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.239553:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b20a2b9a/0xd5401cbabc02262f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b2e expref: 94 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.239556:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026a92 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.239564:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e840 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.239579:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026a92 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.239580:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b20a2b9a/0xd5401cbabc02262f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b2e expref: 94 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.239603:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026a92 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.239608:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502c99c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993284864:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.239629:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.239638:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80840 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.239652:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.239659:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026a92 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.239668:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.239673:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026a92 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.239682:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026a92 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.239697:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026a92 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.239705:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026a92 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8a5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.239728:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.239730:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436342b8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+914:3627:x1870152993282816:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4048us (4134us total) trans 4294976482 rc 0/0 00010000:00010000:1.0:1783520936.239732:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.239737:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.239741:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44980 00000004:00080000:0.0:1783520936.239757:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516311880/2, rc 0, transno 4294976482 00000100:00100000:0.0:1783520936.239771:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516311880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993282816:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.239816:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507939880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993284992:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.239840:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.239873:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993284992 00000100:00100000:3.0:1783520936.239878:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993284992 00000100:00100000:3.0:1783520936.239907:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44988 00000100:00100000:3.0:1783520936.239914:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c9a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+914:3627:x1870152993284992:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520936.239926:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2221 count 1 00010000:00010000:3.0:1783520936.239938:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026aca lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8ad:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.239950:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000516df784/0xd5401cbabc022636 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b3c expref: 94 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.239958:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ad:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.239961:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ad:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.239965:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026aca lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.239985:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000516df784/0xd5401cbabc022636 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b3c expref: 94 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.239994:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000516df784/0xd5401cbabc022636 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b3c expref: 94 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.240005:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b04a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993285120:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.240038:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e780 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.241068:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.241090:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.241113:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.241121:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002484096 00000100:00100000:3.0:1783520936.241127:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002484096 00010000:00010000:3.0:1783520936.241151:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003ec0d447/0xd5401cbabc022613 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2af6 expref: 95 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.241174:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002484352 00000100:00100000:3.0:1783520936.241178:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002484352 00000100:00100000:0.0:1783520936.241188:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002484224 00010000:00010000:3.0:1783520936.241194:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000bba8852e/0xd5401cbabc02260c lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ae8 expref: 96 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.241194:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002484224 00010000:00010000:0.0:1783520936.241214:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000cc8b8804/0xd5401cbabc022628 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b12 expref: 97 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.241220:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36572 00000100:00100000:3.0:1783520936.241277:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c99500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+97:2387:x1870153002484096:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.241277:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36573 00000100:00100000:0.0:1783520936.241289:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36574 00010000:00010000:3.0:1783520936.241292:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.241298:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11ce00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+97:2387:x1870153002484352:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.241301:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f0850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+97:2390:x1870153002484224:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.241303:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003ec0d447/0xd5401cbabc022613 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8a8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2af6 expref: 97 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.241315:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.241322:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.241323:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000cc8b8804/0xd5401cbabc022628 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8aa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b12 expref: 97 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.241329:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9436342b8a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993283456:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.241332:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000bba8852e/0xd5401cbabc02260c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8a7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ae8 expref: 97 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.241353:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a99 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8a7:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.241389:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000bba8852e/0xd5401cbabc02260c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8a7:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ae8 expref: 97 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.241404:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000cc8b8804/0xd5401cbabc022628 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8aa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b12 expref: 97 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.241429:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000bba8852e/0xd5401cbabc02260c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8a7:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ae8 expref: 97 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.241436:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8aa:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.241444:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003ec0d447/0xd5401cbabc022613 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8a8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2af6 expref: 97 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.241445:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8aa:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.241451:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.241454:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc026aae lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8aa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.241461:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.241471:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a99 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.241474:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc026aae lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8aa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.241476:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.241486:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.241509:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a99 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8a7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.241512:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026aa7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.241517:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.241532:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026aa7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8a8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.241547:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.241551:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f0850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+96:2390:x1870153002484224:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 253us (436us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.241552:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.241562:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36574 00010000:00010000:2.0:1783520936.241571:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a99 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.241579:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11ce00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+95:2387:x1870153002484352:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 288us (485us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.241584:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c99500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+95:2387:x1870153002484096:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 311us (515us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.241597:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36573 00010000:00010000:2.0:1783520936.241600:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a99 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.241607:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36572 00010000:00010000:2.0:1783520936.241622:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a99 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.241632:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a99 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.241649:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.241654:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575d180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993284608:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.241660:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.241665:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a99 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.241671:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.241675:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a99 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.241679:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026abc lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.241681:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a99 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.241695:0:9151:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cc8b8804/0xd5401cbabc022628 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8aa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b12 expref: 93 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.241702:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a99 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.241714:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350793bb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993284224:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.241718:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000757ba88a/0xd5401cbabc026a99 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8a7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.241736:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc026aae lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.241748:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc026aae lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.241751:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.241755:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.241758:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.241762:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003ec0d447/0xd5401cbabc022613 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8a8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2af6 expref: 92 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.241767:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc026aae lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.241784:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc026aae lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.241784:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cc7d880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993284736:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.241785:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575e300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993284096:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.241799:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502c99c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993284864:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.241801:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc026aae lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.241806:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026ab5 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.241809:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026aa7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.241827:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026ac3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.241834:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.241835:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026aa7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.241847:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026aa7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.241856:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026aa7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.241862:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026aa7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.241876:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.241881:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc026aae lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.241888:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.241892:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc026aae lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.241901:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc026aae lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.241912:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc026aae lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.241920:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000663c41/0xd5401cbabc026aae lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8aa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.241936:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.241938:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.241941:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.241955:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.241960:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e472300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+916:3625:x1870152993283328:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3807us (4389us total) trans 4294976484 rc 0/0 00000100:00100000:2.0:1783520936.241967:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44982 00010000:00010000:0.0:1783520936.241977:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.241984:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026aa7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.241993:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.241997:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026aa7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.242000:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517247100/2, rc 0, transno 4294976484 00000100:00100000:3.0:1783520936.242003:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350626dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+915:3628:x1870152993283840:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2977us (3136us total) trans 4294976483 rc 0/0 00000100:00100000:2.0:1783520936.242018:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517247100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993283328:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.242019:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026aa7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.242037:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026aa7 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.242045:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026aa7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8a8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.242055:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363573b050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+916:3626:x1870152993283584:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3086us (3312us total) trans 4294976485 rc 0/0 00010000:00010000:0.0:1783520936.242066:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.242070:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.242072:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.242083:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1783520936.242095:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cc7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993285248:0@lo:6:osp-syn-1-0.0 00000004:00080000:2.0:1783520936.242104:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507938380/2, rc 0, transno 4294976483 00000100:00100000:3.0:1783520936.242114:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44985 00000100:00100000:2.0:1783520936.242116:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507938380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993283840:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.242123:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44983 00000100:00100000:0.0:1783520936.242129:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.242160:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b04a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993285120:192.168.204.33@tcp:104: 00000004:00080000:1.0:1783520936.242179:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507938a80/2, rc 0, transno 4294976485 00000100:00100000:2.0:1783520936.242180:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9436342b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993285376:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.242181:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026aca lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.242195:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507938a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993283584:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.242196:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.242219:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.242221:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993285248 00000100:00100000:3.0:1783520936.242227:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993285248 00000100:00100000:1.0:1783520936.242227:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002484608 00000100:00100000:1.0:1783520936.242234:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002484608 00000100:00100000:0.0:1783520936.242234:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.242255:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000841d2395/0xd5401cbabc0228c1 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x8a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fb1 expref: 92 pid: 35276 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.242259:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993285376 00000100:00100000:3.0:1783520936.242265:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993285376 00000100:00100000:0.0:1783520936.242271:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.242291:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002484480 00000100:00100000:1.0:1783520936.242297:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002484480 00000100:00100000:3.0:1783520936.242298:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44989 00000100:00100000:2.0:1783520936.242300:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1783520936.242312:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+916:3627:x1870152993285248:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.242313:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993285504:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.242321:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.242323:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002484736 00000100:00100000:1.0:1783520936.242328:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002484736 00002000:00080000:3.0:1783520936.242335:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2222 count 1 00010000:00010000:1.0:1783520936.242347:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000025a42e3e/0xd5401cbabc02261a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b04 expref: 94 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.242352:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026ad1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8ae:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.242397:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c9a79766/0xd5401cbabc022b1b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f320b expref: 94 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.242403:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.242407:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.242419:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002484864 00000100:00100000:1.0:1783520936.242424:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002484864 00000100:00100000:0.0:1783520936.242435:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002484992 00010000:00010000:1.0:1783520936.242441:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b20a2b9a/0xd5401cbabc02262f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b2e expref: 95 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.242441:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002484992 00010000:00010000:3.0:1783520936.242456:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x8ae:0x0].0x0 since=37s v=(0 0) 00000100:00100000:2.0:1783520936.242457:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36575 00010000:00010000:0.0:1783520936.242461:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000516df784/0xd5401cbabc022636 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b3c expref: 96 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.242464:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ae:0x0].0x0 since=0s v=(1 0) 00000100:00100000:2.0:1783520936.242474:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11f800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+96:2388:x1870153002484608:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.242476:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026ad1 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.242476:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36576 00000100:00100000:1.0:1783520936.242504:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503849180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+96:2387:x1870153002484736:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.242507:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.242518:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000841d2395/0xd5401cbabc0228c1 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x8a6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fb1 expref: 96 pid: 35276 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.242520:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.242526:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36577 00010000:00010000:1.0:1783520936.242531:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000025a42e3e/0xd5401cbabc02261a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8a9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b04 expref: 96 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.242533:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c9a79766/0xd5401cbabc022b1b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f320b expref: 96 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.242541:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000841d2395/0xd5401cbabc0228c1 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8a6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fb1 expref: 96 pid: 35276 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.242541:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517208000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+96:2388:x1870153002484864:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.242558:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.242561:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000025a42e3e/0xd5401cbabc02261a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8a9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b04 expref: 96 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.242566:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c9a79766/0xd5401cbabc022b1b lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f320b expref: 96 pid: 9151 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.242567:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b20a2b9a/0xd5401cbabc02262f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ac:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b2e expref: 96 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.242573:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.242584:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.242594:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000025a42e3e/0xd5401cbabc02261a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8a9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b04 expref: 96 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.242597:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026aa0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.242597:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b20a2b9a/0xd5401cbabc02262f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8ac:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b2e expref: 96 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.242601:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b04a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993285632:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.242623:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.242625:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026aa0 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8a6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.242630:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8a9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.242633:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b20a2b9a/0xd5401cbabc02262f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8ac:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b2e expref: 95 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.242640:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026ab5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.242655:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e780 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.242657:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.242660:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ac:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.242669:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ac:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.242673:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026ab5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8a9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.242675:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026ac3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.242680:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36578 00000100:00100000:2.0:1783520936.242680:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11f800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+94:2388:x1870153002484608:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 211us (487us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.242697:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351720b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+94:2387:x1870153002484992:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.242701:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36575 00010000:00010000:0.0:1783520936.242701:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026ac3 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8ac:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.242713:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.242714:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.242722:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000516df784/0xd5401cbabc022636 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ad:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b3c expref: 93 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.242726:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.242733:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026ab5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.242736:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000516df784/0xd5401cbabc022636 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8ad:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b3c expref: 93 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.242742:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503849180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+93:2387:x1870153002484736:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 251us (470us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.242754:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517208000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+93:2388:x1870153002484864:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 214us (436us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.242757:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026ab5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.242764:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36576 00000100:00100000:0.0:1783520936.242776:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36577 00010000:00010000:2.0:1783520936.242785:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026ab5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.242791:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000516df784/0xd5401cbabc022636 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8ad:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b3c expref: 92 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.242805:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026ab5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.242809:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ad:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.242809:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000126f6394/0xd5401cbabc022621 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b20 expref: 90 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.242817:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ad:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.242828:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026aca lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.242838:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026aca lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8ad:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.242845:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.242851:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.242857:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026ab5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.242859:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.242868:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.242872:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026ab5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.242882:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026ab5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.242892:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993285504 00000100:00100000:3.0:1783520936.242898:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351720b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+90:2387:x1870153002484992:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 203us (498us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.242898:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026ab5 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.242898:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993285504 00010000:00010000:1.0:1783520936.242916:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026ab5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8a9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.242917:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36578 00000100:00100000:0.0:1783520936.242925:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44990 00000100:00100000:3.0:1783520936.242943:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36579 00000100:00100000:2.0:1783520936.242947:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c9b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+918:3627:x1870152993283712:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3953us (4048us total) trans 4294976486 rc 0/0 00010000:00010000:1.0:1783520936.242948:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.242952:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.242957:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520936.242957:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507938700/2, rc 0, transno 4294976486 00000100:00100000:3.0:1783520936.242958:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11ce00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+89:2387:x1870153002484480:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.242962:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44984 00010000:00010000:3.0:1783520936.242972:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.242977:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44991 00000100:00100000:0.0:1783520936.242977:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507938700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993283712:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.242982:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000126f6394/0xd5401cbabc022621 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ab:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b20 expref: 89 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.242990:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b23b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+917:3625:x1870152993285504:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.243001:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000126f6394/0xd5401cbabc022621 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8ab:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b20 expref: 89 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.243005:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2224 count 1 00010000:00010000:1.0:1783520936.243017:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026ad8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8b0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.243025:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993285760:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.243030:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000126f6394/0xd5401cbabc022621 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8ab:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b20 expref: 89 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.243030:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a123a4d9/0xd5401cbabc02264b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b66 expref: 89 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.243052:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b0:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.243057:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ab:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.243057:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.243058:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.243065:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ab:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.243066:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026ad8 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.243072:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026abc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.243081:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993285760 00010000:00010000:3.0:1783520936.243082:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026abc lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8ab:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.243086:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993285760 00010000:00010000:1.0:1783520936.243088:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a123a4d9/0xd5401cbabc02264b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b66 expref: 88 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.243099:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514e9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+918:3625:x1870152993285376:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.243114:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44992 00010000:00010000:1.0:1783520936.243115:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a123a4d9/0xd5401cbabc02264b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b66 expref: 88 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.243117:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00002000:00080000:0.0:1783520936.243120:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2223 count 1 00000100:00100000:2.0:1783520936.243123:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517209500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+918:3627:x1870152993285760:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.243136:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026adf lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8af:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.243138:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575e300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993285888:192.168.204.33@tcp:104: 00002000:00080000:2.0:1783520936.243140:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2225 count 1 00000100:00100000:3.0:1783520936.243151:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11ce00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+88:2387:x1870153002484480:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 195us (916us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.243154:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026ae6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8b1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.243164:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c99d1356/0xd5401cbabc022644 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b58 expref: 88 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.243168:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36579 00010000:00010000:2.0:1783520936.243171:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007082bf89/0xd5401cbabc022659 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b74 expref: 88 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.243184:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.243184:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8af:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.243186:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b1:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.243189:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026aca lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.243194:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8af:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.243195:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.243204:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026abc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.243206:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026adf lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.243207:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026aca lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.243208:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026ae6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.243231:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026abc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.243237:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026aca lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.243246:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026abc lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.243249:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c99d1356/0xd5401cbabc022644 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b58 expref: 87 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.243252:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026aca lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.243252:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007082bf89/0xd5401cbabc022659 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b74 expref: 87 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.243273:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026abc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.243281:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c99d1356/0xd5401cbabc022644 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b58 expref: 87 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.243295:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.243300:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007082bf89/0xd5401cbabc022659 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b74 expref: 87 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.243308:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000100:00100000:2.0:1783520936.243323:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b23480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993286144:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.243323:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1783520936.243329:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026aca lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.243329:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517208e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993286016:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.243336:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.243342:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026abc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.243344:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.243351:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.243351:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026aca lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.243371:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026abc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.243378:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026abc lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.243390:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026abc lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.243396:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80840 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.243404:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026aca lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.243406:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008644bc57/0xd5401cbabc026abc lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8ab:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.243423:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c9a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+919:3627:x1870152993284992:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3506us (3579us total) trans 4294976487 rc 0/0 00010000:00010000:1.0:1783520936.243430:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026aca lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.243431:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507939880/2, rc 0, transno 4294976487 00000100:00100000:3.0:1783520936.243435:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44988 00010000:00010000:2.0:1783520936.243435:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.243440:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.243444:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.243446:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507939880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993284992:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.243459:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000bf1249c3/0xd5401cbabc026aca lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8ad:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.243477:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.243480:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.243481:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000841d2395/0xd5401cbabc0228c1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8a6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->4198399) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fb1 expref: 87 pid: 35276 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.243484:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.243503:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b16c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.243513:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350793ad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993284480:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.243523:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032e9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993286272:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.243538:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026aa0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.243538:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026ac3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.243562:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.243564:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026aa0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.243564:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026ac3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.243584:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026aa0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.243588:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026ac3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.243597:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026aa0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.243603:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026ac3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.243604:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993286272 00010000:00010000:3.0:1783520936.243607:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026aa0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.243612:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993286272 00000100:00100000:1.0:1783520936.243621:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+920:3625:x1870152993283968:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4357us (4586us total) trans 4294976488 rc 0/0 00000100:00100000:1.0:1783520936.243633:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44986 00000004:00080000:2.0:1783520936.243635:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350793aa00/2, rc 0, transno 4294976488 00010000:00010000:0.0:1783520936.243635:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:0.0:1783520936.243650:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00000100:00100000:2.0:1783520936.243651:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350793aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993283968:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.243657:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.243658:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026ac3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.243665:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026aa0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.243670:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbebe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.243676:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026ac3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.243679:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.243685:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026aa0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.243695:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993286400:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.243710:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to regular blp list, 0 blwis in pool 00010000:00010000:1.0:1783520936.243712:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026aa0 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.243713:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026ac3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.243736:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.243743:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026ac3 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.243745:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026aa0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.243762:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026ac3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8ac:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.243765:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000051552066/0xd5401cbabc026aa0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8a6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.243788:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993286400 00010000:00010000:0.0:1783520936.243791:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.243793:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993286400 00010000:00010000:0.0:1783520936.243795:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.243800:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.243804:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.243808:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.243810:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.243829:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350626d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+921:3628:x1870152993283200:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5714us (6469us total) trans 4294976489 rc 0/0 00000100:00100000:1.0:1783520936.243839:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44993 00000004:00080000:2.0:1783520936.243843:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516310700/2, rc 0, transno 4294976489 00000100:00100000:3.0:1783520936.243849:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44981 00000100:00100000:1.0:1783520936.243854:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b21f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+922:3628:x1870152993286272:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.243855:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516310700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993283200:0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.243874:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2226 count 1 00000100:00100000:0.0:1783520936.243898:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351cc7d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+921:3626:x1870152993284352:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4490us (4587us total) trans 4294976490 rc 0/0 00000100:00100000:1.0:1783520936.243899:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e0f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993286528:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.243912:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44994 00000100:00100000:0.0:1783520936.243921:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44987 00000100:00100000:2.0:1783520936.243923:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e0f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+921:3625:x1870152993286400:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.243938:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2227 count 1 00000100:00100000:1.0:1783520936.243944:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.243955:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026aed lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8b3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.243969:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e297e860/0xd5401cbabc022660 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b90 expref: 86 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.243977:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507939c00/2, rc 0, transno 4294976490 00010000:00010000:2.0:1783520936.243978:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b3:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.243985:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.243992:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026aed lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.243992:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507939c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993284352:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.244011:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026af4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8b2:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.244022:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000045bff545/0xd5401cbabc022652 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b82 expref: 86 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.244032:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b2:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.244036:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.244040:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026af4 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.244055:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:1.0:1783520936.244065:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000045bff545/0xd5401cbabc022652 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b82 expref: 86 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.244068:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e0f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993286656:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.244083:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993286528 00000100:00100000:0.0:1783520936.244088:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993286528 00010000:00010000:1.0:1783520936.244089:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000045bff545/0xd5401cbabc022652 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b82 expref: 86 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.244101:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575ca80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993286784:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.244108:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.244111:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44995 00000100:00100000:0.0:1783520936.244121:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+921:3626:x1870152993286528:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.244133:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2228 count 1 00000100:00100000:1.0:1783520936.244143:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470480 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.244146:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026afb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8b4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.244153:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e297e860/0xd5401cbabc022660 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b90 expref: 86 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.244161:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003daa3566/0xd5401cbabc022667 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b9e expref: 86 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.244170:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e297e860/0xd5401cbabc022660 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b90 expref: 86 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.244183:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b4:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.244190:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b4:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.244195:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943505876d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993286912:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.244195:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026afb lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.244225:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003daa3566/0xd5401cbabc022667 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b9e expref: 86 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.244233:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b19c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.244235:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003daa3566/0xd5401cbabc022667 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b9e expref: 86 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.244247:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993286656 00000100:00100000:0.0:1783520936.244248:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032e8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993287040:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.244251:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993286656 00000100:00100000:3.0:1783520936.244277:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44996 00000100:00100000:0.0:1783520936.244281:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80900 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.244290:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505875c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+922:3628:x1870152993286656:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520936.244305:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2229 count 1 00010000:00010000:3.0:1783520936.244318:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026b02 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8b5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.244331:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006fd6c9df/0xd5401cbabc02266e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bac expref: 86 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.244339:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b5:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.244343:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.244347:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026b02 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.244390:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006fd6c9df/0xd5401cbabc02266e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bac expref: 86 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.244402:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006fd6c9df/0xd5401cbabc02266e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bac expref: 86 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.244415:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b05c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993287168:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.244558:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e840 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.244617:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b04a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993285632:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.244632:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026ad1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.244913:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575e300 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993285888:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.244933:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026ad8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.245171:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517208e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993286016:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.245180:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.245188:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026adf lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.245206:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002485120 00000100:00100000:0.0:1783520936.245210:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002485120 00010000:00010000:0.0:1783520936.245229:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c9a79766/0xd5401cbabc022b1b lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f320b expref: 87 pid: 9151 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.245255:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36580 00000100:00100000:0.0:1783520936.245264:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032eb800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+87:2388:x1870153002485120:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.245273:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.245278:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c9a79766/0xd5401cbabc022b1b lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8ae:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f320b expref: 87 pid: 9151 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.245288:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b23480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993286144:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.245304:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c9a79766/0xd5401cbabc022b1b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8ae:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f320b expref: 87 pid: 9151 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.245310:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026ae6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.245322:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c9a79766/0xd5401cbabc022b1b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8ae:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 50331648->50335743) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f320b expref: 87 pid: 9151 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.245333:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ae:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.245336:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ae:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.245340:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026ad1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.245348:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026ad1 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8ae:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.245389:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.245402:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026ad1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.245413:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026ad1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.245420:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032eb800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+86:2388:x1870153002485120:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 156us (241us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.245426:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026ad1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.245431:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36580 00010000:00010000:3.0:1783520936.245441:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026ad1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.245460:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.245468:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.245474:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026ad1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.245481:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.245486:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026ad1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.245511:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026ad1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.245523:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026ad1 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.245530:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000fa48b86/0xd5401cbabc026ad1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.245544:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.245546:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.245548:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.245600:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+923:3627:x1870152993285248:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3291us (3472us total) trans 4294976491 rc 0/0 00000100:00100000:3.0:1783520936.245608:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44989 00000004:00080000:0.0:1783520936.245613:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cc7d880/2, rc 0, transno 4294976491 00000100:00100000:0.0:1783520936.245626:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cc7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993285248:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.245678:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943505874000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993287296:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.245710:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.245735:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993287296 00000100:00100000:3.0:1783520936.245740:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993287296 00000100:00100000:3.0:1783520936.245766:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44997 00000100:00100000:3.0:1783520936.245772:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351664a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+923:3628:x1870152993287296:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520936.245784:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2230 count 1 00010000:00010000:3.0:1783520936.245796:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026b09 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8b6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.245813:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000646dbb1f/0xd5401cbabc022a0a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f30fa expref: 85 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.245820:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x8b6:0x0].0x0 since=37s v=(0 0) 00010000:00010000:3.0:1783520936.245823:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.245828:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026b09 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.245846:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000646dbb1f/0xd5401cbabc022a0a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30fa expref: 85 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.245854:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000646dbb1f/0xd5401cbabc022a0a lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30fa expref: 85 pid: 178496 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.245864:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b04a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993287424:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.245899:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.246524:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575ca80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993286784:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.246543:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026af4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.246589:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.246610:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.246629:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002485376 00000100:00100000:0.0:1783520936.246634:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.246635:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002485376 00000100:00100000:0.0:1783520936.246661:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.246662:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007082bf89/0xd5401cbabc022659 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b74 expref: 86 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.246690:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002485632 00000100:00100000:3.0:1783520936.246694:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032e8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993287040:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.246695:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002485632 00010000:00010000:1.0:1783520936.246708:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000045bff545/0xd5401cbabc022652 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b82 expref: 87 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.246712:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026afb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.246726:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.246739:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002485888 00000100:00100000:1.0:1783520936.246743:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002485888 00000100:00100000:3.0:1783520936.246754:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b05c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993287168:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.246762:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003daa3566/0xd5401cbabc022667 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b9e expref: 88 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.246763:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.246767:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026b02 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.246783:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.246788:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002485504 00000100:00100000:1.0:1783520936.246794:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002485504 00010000:00010000:1.0:1783520936.246810:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c99d1356/0xd5401cbabc022644 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b58 expref: 89 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.246824:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002485760 00000100:00100000:0.0:1783520936.246826:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002486016 00000100:00100000:1.0:1783520936.246827:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002485760 00000100:00100000:0.0:1783520936.246831:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002486016 00010000:00010000:1.0:1783520936.246841:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e297e860/0xd5401cbabc022660 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b90 expref: 91 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.246853:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006fd6c9df/0xd5401cbabc02266e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bac expref: 91 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.246858:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002485248 00000100:00100000:1.0:1783520936.246864:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002485248 00000100:00100000:0.0:1783520936.246880:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36581 00000100:00100000:3.0:1783520936.246885:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b04a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993287424:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.246885:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a123a4d9/0xd5401cbabc02264b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b66 expref: 92 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.246894:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f1c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+92:2389:x1870153002485376:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.246901:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026b09 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.246908:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.246917:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007082bf89/0xd5401cbabc022659 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b74 expref: 92 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.246921:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36582 00010000:00010000:0.0:1783520936.246935:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007082bf89/0xd5401cbabc022659 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8b1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b74 expref: 92 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.246936:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351664bb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+92:2389:x1870153002485632:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.246958:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007082bf89/0xd5401cbabc022659 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8b1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b74 expref: 92 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.246964:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.246968:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.246973:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000045bff545/0xd5401cbabc022652 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b82 expref: 91 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.246974:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.246982:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026ae6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.246988:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000045bff545/0xd5401cbabc022652 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8b2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b82 expref: 91 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.246997:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026ae6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8b1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.247005:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000045bff545/0xd5401cbabc022652 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8b2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b82 expref: 91 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.247015:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.247019:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.247019:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.247025:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026af4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.247033:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026af4 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8b2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.247045:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f1c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+90:2389:x1870153002485376:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 155us (456us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.247055:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.247057:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36581 00000100:00100000:1.0:1783520936.247092:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351664bb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+89:2389:x1870153002485632:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 159us (480us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.247098:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36582 00000100:00100000:0.0:1783520936.247109:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36583 00000100:00100000:1.0:1783520936.247116:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36584 00000100:00100000:0.0:1783520936.247122:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351664aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+88:2389:x1870153002485888:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.247128:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351664b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+88:2389:x1870153002485504:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.247130:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.247136:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.247138:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003daa3566/0xd5401cbabc022667 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b9e expref: 88 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.247143:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c99d1356/0xd5401cbabc022644 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8af:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b58 expref: 88 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.247160:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003daa3566/0xd5401cbabc022667 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8b4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b9e expref: 88 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.247186:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c99d1356/0xd5401cbabc022644 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8af:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b58 expref: 88 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.247193:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003daa3566/0xd5401cbabc022667 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8b4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b9e expref: 88 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.247200:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c99d1356/0xd5401cbabc022644 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8af:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b58 expref: 88 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.247206:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.247210:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8af:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.247211:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.247218:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8af:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.247221:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026afb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.247223:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026adf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.247245:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026adf lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8af:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.247245:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026afb lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8b4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.247271:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.247278:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.247285:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943505876d80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993286912:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.247288:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351664b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+86:2389:x1870153002485504:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 162us (628us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.247295:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36584 00010000:00010000:2.0:1783520936.247309:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026aed lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.247309:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351664aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+85:2389:x1870153002485888:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 187us (671us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.247310:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026adf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.247310:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36585 00000100:00100000:1.0:1783520936.247322:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516649500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+85:2389:x1870153002485760:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.247331:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36583 00010000:00010000:1.0:1783520936.247335:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.247339:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026adf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.247342:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e297e860/0xd5401cbabc022660 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b90 expref: 85 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.247382:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e297e860/0xd5401cbabc022660 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8b3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b90 expref: 84 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.247385:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026adf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.247388:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36586 00010000:00010000:1.0:1783520936.247394:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e297e860/0xd5401cbabc022660 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8b3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b90 expref: 84 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.247404:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.247408:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026adf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.247414:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b3:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.247417:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002486144 00010000:00010000:1.0:1783520936.247418:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.247423:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026aed lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.247423:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002486144 00010000:00010000:1.0:1783520936.247432:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026aed lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8b3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.247434:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.247443:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000646dbb1f/0xd5401cbabc022a0a lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30fa expref: 84 pid: 178496 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.247448:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.247455:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026adf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.247462:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.247465:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026adf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.247469:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36587 00010000:00010000:1.0:1783520936.247476:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026adf lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.247483:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509175880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+84:2388:x1870153002485248:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.247508:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.247514:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026adf lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.247515:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a123a4d9/0xd5401cbabc02264b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b66 expref: 84 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.247536:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026adf lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.247542:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a123a4d9/0xd5401cbabc02264b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8b0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b66 expref: 84 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.247563:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.247564:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a123a4d9/0xd5401cbabc02264b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8b0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2b66 expref: 84 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.247568:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.247572:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.247582:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.247586:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b0:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.247590:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36588 00010000:00010000:0.0:1783520936.247591:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026ad8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.247602:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026ad8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8b0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.247604:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f1450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+83:2388:x1870153002486144:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.247619:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.247626:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000646dbb1f/0xd5401cbabc022a0a lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8b6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30fa expref: 83 pid: 178496 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.247635:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.247637:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514e9ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+924:3625:x1870152993285376:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4698us (5415us total) trans 4294976492 rc 0/0 00010000:00010000:1.0:1783520936.247643:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000646dbb1f/0xd5401cbabc022a0a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8b6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30fa expref: 83 pid: 178496 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.247652:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44990 00000004:00080000:2.0:1783520936.247657:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9436342b8a80/2, rc 0, transno 4294976492 00010000:00010000:1.0:1783520936.247664:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000646dbb1f/0xd5401cbabc022a0a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8b6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 29360128->29364223) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f30fa expref: 83 pid: 178496 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.247672:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9436342b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993285376:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.247674:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b6:0x0].0x0 since=0s v=(1 0) 00000100:00100000:0.0:1783520936.247674:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509175880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+82:2388:x1870153002485248:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 191us (889us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.247681:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026afb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.247682:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.247690:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026b09 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.247693:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36587 00010000:00010000:1.0:1783520936.247701:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026b09 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8b6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.247719:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026afb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.247721:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509177b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+81:2389:x1870153002486016:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.247723:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.247735:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.247736:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026afb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.247737:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943505876d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993287552:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.247747:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006fd6c9df/0xd5401cbabc02266e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bac expref: 81 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.247748:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f1450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+81:2388:x1870153002486144:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 145us (345us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.247760:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026afb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.247764:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36588 00010000:00010000:0.0:1783520936.247775:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006fd6c9df/0xd5401cbabc02266e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8b5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bac expref: 81 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.247776:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.247796:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006fd6c9df/0xd5401cbabc02266e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8b5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bac expref: 80 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.247797:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.247800:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.247812:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.247815:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.247819:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026b02 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.247828:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516649500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+79:2389:x1870153002485760:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 504us (1102us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.247829:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993287552 00010000:00010000:0.0:1783520936.247830:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026b02 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8b5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.247836:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993287552 00000100:00100000:1.0:1783520936.247848:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36585 00010000:00010000:0.0:1783520936.247855:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.247862:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026afb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.247870:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44998 00010000:00010000:0.0:1783520936.247873:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.247877:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026ae6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.247878:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026afb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.247882:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505875f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+925:3625:x1870152993287552:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.247903:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026ae6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.247904:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026afb lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.247905:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+925:3626:x1870152993286528:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3784us (3963us total) trans 4294976493 rc 0/0 00000004:00080000:2.0:1783520936.247914:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e0f6300/2, rc 0, transno 4294976493 00000100:00100000:3.0:1783520936.247925:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44995 00000100:00100000:2.0:1783520936.247929:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e0f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993286528:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.247929:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026ae6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.247932:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026afb lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.247949:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026ae6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.247952:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006cf32aef/0xd5401cbabc026afb lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8b4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.247953:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026b09 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.247964:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1783520936.247975:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943505876680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993287680:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.247979:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026b09 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.247981:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.247990:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.247993:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026b09 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.247995:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.247995:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026ae6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.248004:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.248007:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026af4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.248018:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026b09 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.248020:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.248027:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026ae6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.248029:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026af4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.248040:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993287680 00010000:00010000:3.0:1783520936.248045:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.248046:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993287680 00010000:00010000:0.0:1783520936.248053:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026ae6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.248055:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026af4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.248075:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 44999 00010000:00010000:1.0:1783520936.248077:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026af4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.248089:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505874e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+926:3625:x1870152993287680:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.248092:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026ae6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.248099:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:0.0:1783520936.248103:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026ae6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.248109:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00002000:00080000:2.0:1783520936.248110:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2232 count 1 00010000:00010000:1.0:1783520936.248116:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026b09 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.248119:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.248127:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026af4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.248128:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026b10 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8b8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.248130:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4e00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:3.0:1783520936.248131:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351664a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+926:3628:x1870152993287296:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2355us (2419us total) trans 4294976494 rc 0/0 00010000:00010000:1.0:1783520936.248138:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026b09 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.248152:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.248160:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026af4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.248161:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44997 00010000:00010000:1.0:1783520936.248161:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026b09 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.248166:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000491d36e0/0xd5401cbabc022683 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bd6 expref: 78 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.248184:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026af4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.248190:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b8:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.248193:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026b09 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.248198:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.248200:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026af4 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.248215:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026b10 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.248225:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000df41884b/0xd5401cbabc026af4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.248231:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026b09 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8b6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.248257:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.248259:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000491d36e0/0xd5401cbabc022683 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bd6 expref: 78 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.248261:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.248264:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.248266:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.248269:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.248274:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.248276:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.248280:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000491d36e0/0xd5401cbabc022683 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bd6 expref: 78 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.248294:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943505877800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993287808:192.168.204.33@tcp:104: 00000004:00080000:0.0:1783520936.248301:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943505874000/2, rc 0, transno 4294976494 00000100:00100000:0.0:1783520936.248317:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943505874000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993287296:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.248337:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b19c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.248350:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.248353:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026aed lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.248412:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517209500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+926:3627:x1870152993285760:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5290us (5358us total) trans 4294976495 rc 0/0 00010000:00010000:3.0:1783520936.248417:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026b02 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.248421:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44992 00000100:00100000:0.0:1783520936.248427:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509177b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+78:2389:x1870153002486016:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 1025us (1663us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.248432:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026b02 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.248439:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36586 00010000:00010000:2.0:1783520936.248445:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026aed lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.248454:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026b02 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.248454:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993287936:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.248457:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026aed lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.248479:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026b02 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.248479:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026aed lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.248508:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.248513:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1783520936.248525:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520936.248533:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520936.248535:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.248539:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026aed lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.248541:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026b02 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.248561:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.248563:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.248568:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026aed lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.248569:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026b02 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.248594:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026aed lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.248594:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026b02 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.248623:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e0f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+927:3625:x1870152993286400:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4702us (4890us total) trans 4294976496 rc 0/0 00010000:00010000:0.0:1783520936.248628:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026b02 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.248629:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026aed lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.248630:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505875c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+927:3628:x1870152993286656:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4344us (4524us total) trans 4294976497 rc 0/0 00000100:00100000:2.0:1783520936.248632:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44994 00000100:00100000:3.0:1783520936.248652:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44996 00010000:00010000:0.0:1783520936.248653:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e5f2f058/0xd5401cbabc026b02 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8b5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.248662:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513b22d80/2, rc 0, transno 4294976496 00000100:00100000:3.0:1783520936.248672:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993287936 00010000:00010000:1.0:1783520936.248673:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026aed lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.248678:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993287936 00000100:00100000:2.0:1783520936.248679:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993286400:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.248682:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.248686:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.248692:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.248706:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.248710:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.248711:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45000 00010000:00010000:1.0:1783520936.248715:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520936.248719:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e0f6d80/2, rc 0, transno 4294976497 00000100:00100000:3.0:1783520936.248723:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+926:3627:x1870152993287936:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.248737:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e0f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993286656:0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520936.248739:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2233 count 1 00000004:00080000:1.0:1783520936.248742:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513b20a80/2, rc 0, transno 4294976495 00002000:00080000:2.0:1783520936.248745:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2231 count 1 00000100:00100000:1.0:1783520936.248752:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993285760:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.248754:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517208000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993288064:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.248756:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026b17 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8b9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.248760:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026b1e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8b7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.248783:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000779de112/0xd5401cbabc02268a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2be4 expref: 77 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.248784:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000088f158fa/0xd5401cbabc02267c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bc8 expref: 77 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.248792:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.248806:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b9:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.248807:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b7:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.248813:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.248815:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.248821:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026b17 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.248822:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026b1e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.248852:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000779de112/0xd5401cbabc02268a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2be4 expref: 77 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.248860:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509177b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993288192:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.248862:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000779de112/0xd5401cbabc02268a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2be4 expref: 77 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.248862:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b21f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+927:3628:x1870152993286272:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5012us (5300us total) trans 4294976498 rc 0/0 00000100:00100000:1.0:1783520936.248880:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44993 00000100:00100000:3.0:1783520936.248883:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b05c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993288320:192.168.204.33@tcp:104: 00000004:00080000:0.0:1783520936.248888:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435032e9500/2, rc 0, transno 4294976498 00000100:00100000:0.0:1783520936.248900:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032e9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993286272:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.248901:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.248902:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993288064 00000100:00100000:1.0:1783520936.248910:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993288064 00000100:00100000:3.0:1783520936.248927:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e780 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.248936:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1783520936.248942:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993288192 00000100:00100000:2.0:1783520936.248943:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509177100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993288448:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.248950:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993288192 00000100:00100000:3.0:1783520936.248952:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45001 00000100:00100000:0.0:1783520936.248958:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1783520936.248963:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509175880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+928:3628:x1870152993288064:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.248971:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.248972:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509175500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993288576:0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520936.248977:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2234 count 1 00010000:00010000:1.0:1783520936.248981:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026ad8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.248989:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026b25 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8ba:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.248992:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026ad8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.248997:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993288448 00000100:00100000:2.0:1783520936.249003:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993288448 00000100:00100000:0.0:1783520936.249010:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.249017:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026ad8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.249018:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001db948db/0xd5401cbabc022691 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bf2 expref: 77 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.249028:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993288576 00000100:00100000:2.0:1783520936.249032:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993288576 00010000:00010000:1.0:1783520936.249040:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026ad8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.249041:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ba:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.249050:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ba:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.249056:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45002 00010000:00010000:3.0:1783520936.249061:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026b25 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.249070:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517208e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+930:3625:x1870152993288448:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.249079:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00002000:00080000:2.0:1783520936.249087:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2236 count 1 00010000:00010000:1.0:1783520936.249090:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.249094:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001db948db/0xd5401cbabc022691 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bf2 expref: 77 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.249097:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026ad8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.249101:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026b2c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8bc:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.249118:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001db948db/0xd5401cbabc022691 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bf2 expref: 77 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.249122:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.249129:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026ad8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.249131:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ccc82b4d/0xd5401cbabc02269f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c0e expref: 77 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.249142:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b07100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993288704:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.249153:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8bc:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.249155:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026ad8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.249156:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8bc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.249164:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026b2c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.249172:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.249178:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026ad8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.249189:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026ad8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8b0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.249190:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ccc82b4d/0xd5401cbabc02269f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c0e expref: 77 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.249211:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ccc82b4d/0xd5401cbabc02269f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c0e expref: 77 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.249215:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.249218:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.249223:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.249228:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45003 00000100:00100000:2.0:1783520936.249228:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e3b0a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993288832:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.249237:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509174a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+930:3628:x1870152993288576:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520936.249248:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2237 count 1 00000100:00100000:2.0:1783520936.249258:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1300 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.249264:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026b33 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8bd:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.249273:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000066daea04/0xd5401cbabc0226a6 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c1c expref: 77 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.249278:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8bd:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.249280:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8bd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.249283:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026b33 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.249300:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000066daea04/0xd5401cbabc0226a6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c1c expref: 77 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.249309:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000066daea04/0xd5401cbabc0226a6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c1c expref: 77 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.249317:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b23b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+931:3625:x1870152993285504:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6330us (6917us total) trans 4294976499 rc 0/0 00000100:00100000:3.0:1783520936.249318:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032eb800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993288960:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.249329:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44991 00000004:00080000:2.0:1783520936.249337:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513b21180/2, rc 0, transno 4294976499 00000100:00100000:2.0:1783520936.249348:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993285504:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.249395:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45004 00000100:00100000:2.0:1783520936.249403:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351720b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+930:3625:x1870152993288192:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.249416:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2235 count 1 00010000:00010000:2.0:1783520936.249427:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026b3a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8bb:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.249429:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e840 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.249447:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c6e1a2e1/0xd5401cbabc022698 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c00 expref: 77 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.249452:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8bb:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.249455:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8bb:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.249456:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:2.0:1783520936.249458:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026b3a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.249468:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509176a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993289088:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.249477:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c6e1a2e1/0xd5401cbabc022698 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c00 expref: 77 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.249485:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c6e1a2e1/0xd5401cbabc022698 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c00 expref: 77 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.249508:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e3b1880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993289216:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.249513:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.249537:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1180 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.249539:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993289088 00000100:00100000:0.0:1783520936.249544:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993289088 00010000:00010000:2.0:1783520936.249562:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000088f158fa/0xd5401cbabc02267c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bc8 expref: 77 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.249569:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000088f158fa/0xd5401cbabc02267c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bc8 expref: 77 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.249578:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e3b0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993289344:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.249584:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45005 00000100:00100000:1.0:1783520936.249593:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+931:3627:x1870152993289088:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.249602:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1a80 going to sleep for 11 seconds 00002000:00080000:1.0:1783520936.249605:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2238 count 1 00010000:00010000:1.0:1783520936.249617:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026b41 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8be:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.249627:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000000efada50/0xd5401cbabc0228f9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fe9 expref: 77 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.249632:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x8be:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.249634:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8be:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.249636:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026b41 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.249652:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000000efada50/0xd5401cbabc0228f9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fe9 expref: 77 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.249658:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000000efada50/0xd5401cbabc0228f9 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fe9 expref: 77 pid: 8409 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.249664:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b23480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993289472:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.249688:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470a80 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.250252:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.250270:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943505877800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993287808:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.250283:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026b10 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.250298:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002486272 00000100:00100000:1.0:1783520936.250302:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002486272 00010000:00010000:1.0:1783520936.250318:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000491d36e0/0xd5401cbabc022683 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bd6 expref: 78 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.250343:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36589 00000100:00100000:1.0:1783520936.250350:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351664bb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+78:2390:x1870153002486272:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.250381:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.250388:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000491d36e0/0xd5401cbabc022683 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bd6 expref: 78 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.250403:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000491d36e0/0xd5401cbabc022683 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8b8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bd6 expref: 78 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.250419:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000491d36e0/0xd5401cbabc022683 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8b8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bd6 expref: 78 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.250429:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.250432:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.250437:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026b10 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.250445:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026b10 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8b8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.250463:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.250482:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026b10 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.250504:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026b10 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.250505:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351664bb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+77:2390:x1870153002486272:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 153us (254us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.250521:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026b10 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.250521:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36589 00010000:00010000:2.0:1783520936.250531:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026b10 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.250550:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.250634:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505874e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+932:3625:x1870152993287680:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2547us (2632us total) trans 4294976500 rc 0/0 00000100:00100000:2.0:1783520936.250641:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44999 00000004:00080000:0.0:1783520936.250646:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943505876680/2, rc 0, transno 4294976500 00000100:00100000:0.0:1783520936.250661:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943505876680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993287680:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.250678:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.250685:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026b10 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.250694:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.250697:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026b10 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.250704:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026b10 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.250714:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026b10 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.250719:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026b10 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8b8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.250722:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514dd6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993289600:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.250735:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.250738:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.250740:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.250752:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.250786:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993289600 00000100:00100000:1.0:1783520936.250791:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993289600 00000100:00100000:1.0:1783520936.250818:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45006 00000100:00100000:1.0:1783520936.250825:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351664b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+932:3625:x1870152993289600:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.250839:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2239 count 1 00010000:00010000:1.0:1783520936.250851:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026b48 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8bf:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.250864:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000000d98bf22/0xd5401cbabc0226b4 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c38 expref: 76 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.250871:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8bf:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.250874:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8bf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.250878:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026b48 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.250897:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000000d98bf22/0xd5401cbabc0226b4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c38 expref: 76 pid: 20415 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.250907:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000000d98bf22/0xd5401cbabc0226b4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c38 expref: 76 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.250917:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514dd7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993289728:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.250953:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470000 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.251913:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b07100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993288704:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.251933:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026b25 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.251937:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b05c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993288320:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.251943:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.251956:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032eb800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993288960:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.251958:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026b17 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.251969:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002486528 00000100:00100000:1.0:1783520936.251974:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002486528 00010000:00010000:2.0:1783520936.251975:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026b33 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.252003:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e3b0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993289344:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.252015:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.252017:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026b1e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.252023:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001db948db/0xd5401cbabc022691 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bf2 expref: 77 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.252036:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.252044:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e3b1880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993289216:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.252054:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002486400 00010000:00010000:2.0:1783520936.252058:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026b3a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.252058:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002486400 00000100:00100000:0.0:1783520936.252070:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.252082:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000779de112/0xd5401cbabc02268a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2be4 expref: 78 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.252093:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.252114:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002486784 00000100:00100000:1.0:1783520936.252115:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002487040 00000100:00100000:2.0:1783520936.252117:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002486784 00000100:00100000:1.0:1783520936.252120:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002487040 00000100:00100000:0.0:1783520936.252130:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.252135:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c6e1a2e1/0xd5401cbabc022698 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c00 expref: 80 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.252140:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000000efada50/0xd5401cbabc0228f9 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x8be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fe9 expref: 80 pid: 8409 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.252156:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002486656 00000100:00100000:2.0:1783520936.252159:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002486912 00000100:00100000:3.0:1783520936.252161:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002486656 00000100:00100000:1.0:1783520936.252161:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36590 00000100:00100000:2.0:1783520936.252165:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002486912 00000100:00100000:1.0:1783520936.252176:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351664aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+82:2387:x1870153002486528:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.252182:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000066daea04/0xd5401cbabc0226a6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c1c expref: 82 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.252184:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000088f158fa/0xd5401cbabc02267c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bc8 expref: 82 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.252186:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.252188:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.252198:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001db948db/0xd5401cbabc022691 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ba:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bf2 expref: 82 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.252215:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001db948db/0xd5401cbabc022691 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8ba:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bf2 expref: 82 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.252219:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002487168 00000100:00100000:2.0:1783520936.252224:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36591 00000100:00100000:3.0:1783520936.252225:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002487168 00000100:00100000:0.0:1783520936.252234:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.252235:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516649500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+83:2390:x1870153002486400:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.252239:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001db948db/0xd5401cbabc022691 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8ba:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bf2 expref: 83 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.252244:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000000d98bf22/0xd5401cbabc0226b4 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c38 expref: 83 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.252245:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.252253:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002487296 00010000:00010000:2.0:1783520936.252255:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000779de112/0xd5401cbabc02268a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2be4 expref: 83 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.252258:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002487296 00010000:00010000:1.0:1783520936.252259:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ba:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.252268:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ba:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.252277:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ccc82b4d/0xd5401cbabc02269f lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c0e expref: 83 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.252278:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026b25 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.252282:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000779de112/0xd5401cbabc02268a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8b9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2be4 expref: 83 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.252285:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36592 00000100:00100000:3.0:1783520936.252298:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503849180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+83:2390:x1870153002486784:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.252306:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026b25 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.252308:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36593 00010000:00010000:3.0:1783520936.252309:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.252318:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c6e1a2e1/0xd5401cbabc022698 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8bb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c00 expref: 83 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.252322:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11f800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+83:2388:x1870153002487040:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.252324:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000779de112/0xd5401cbabc02268a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8b9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2be4 expref: 83 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.252337:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.252341:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.252344:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.252347:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c6e1a2e1/0xd5401cbabc022698 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8bb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c00 expref: 82 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.252347:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000000efada50/0xd5401cbabc0228f9 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x8be:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fe9 expref: 82 pid: 8409 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.252349:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.252381:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026b17 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.252389:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000000efada50/0xd5401cbabc0228f9 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8be:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fe9 expref: 82 pid: 8409 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.252395:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026b17 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.252397:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351664aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+82:2387:x1870153002486528:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 223us (457us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.252399:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c6e1a2e1/0xd5401cbabc022698 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8bb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c00 expref: 82 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.252409:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8be:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.252413:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8be:0x0].0x0 since=0s v=(1 0) 00000100:00100000:1.0:1783520936.252415:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36590 00010000:00010000:0.0:1783520936.252419:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026b41 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8be:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.252425:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.252429:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026b41 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8be:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.252438:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36594 00010000:00010000:0.0:1783520936.252444:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.252446:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516649500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+81:2390:x1870153002486400:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 212us (429us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.252451:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11ce00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+81:2390:x1870153002486656:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.252463:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11f800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+81:2388:x1870153002487040:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 144us (389us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.252465:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36591 00010000:00010000:1.0:1783520936.252466:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.252475:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000066daea04/0xd5401cbabc0226a6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8bd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c1c expref: 81 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.252482:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36593 00000100:00100000:2.0:1783520936.252499:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36595 00010000:00010000:1.0:1783520936.252508:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000066daea04/0xd5401cbabc0226a6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8bd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c1c expref: 79 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.252509:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f3850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+79:2390:x1870153002486912:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.252509:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36596 00010000:00010000:2.0:1783520936.252518:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.252519:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e006d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+79:2387:x1870153002487168:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.252525:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000088f158fa/0xd5401cbabc02267c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8b7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bc8 expref: 79 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.252533:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.252537:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8bb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.252538:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000066daea04/0xd5401cbabc0226a6 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8bd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c1c expref: 78 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.252540:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000000d98bf22/0xd5401cbabc0226b4 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8bf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c38 expref: 78 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.252541:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000088f158fa/0xd5401cbabc02267c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8b7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bc8 expref: 78 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.252546:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8bb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.252557:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026b3a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.252568:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000000d98bf22/0xd5401cbabc0226b4 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8bf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c38 expref: 78 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.252585:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026b3a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.252590:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8bf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.252592:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000088f158fa/0xd5401cbabc02267c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8b7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2bc8 expref: 77 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.252594:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8bd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.252597:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8bf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.252606:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8bd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.252608:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026b48 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8bf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.252612:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.252613:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026b33 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.252617:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8b7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.252621:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.252621:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026b48 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8bf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.252627:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026b1e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.252639:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026b33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.252642:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026b1e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8b7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.252647:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503849180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+76:2390:x1870153002486784:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 349us (609us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.252655:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.252666:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36592 00000100:00100000:0.0:1783520936.252677:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e006d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+76:2387:x1870153002487168:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 159us (490us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.252681:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.252683:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.252685:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36596 00000100:00100000:3.0:1783520936.252695:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36597 00000100:00100000:3.0:1783520936.252706:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503325c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+74:2390:x1870153002487296:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.252711:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11ce00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+74:2390:x1870153002486656:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 262us (616us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.252713:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f3850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+74:2390:x1870153002486912:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 204us (581us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.252723:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e3b0a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993288832:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.252727:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.252728:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36594 00010000:00010000:3.0:1783520936.252737:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ccc82b4d/0xd5401cbabc02269f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8bc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c0e expref: 74 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.252738:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026b2c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8bc:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.252741:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36595 00010000:00010000:1.0:1783520936.252759:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026b25 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.252766:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026b25 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.252769:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ccc82b4d/0xd5401cbabc02269f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8bc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c0e expref: 72 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.252777:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026b25 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.252786:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026b25 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.252791:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026b33 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.252794:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ccc82b4d/0xd5401cbabc02269f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8bc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c0e expref: 72 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.252808:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.252810:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026b33 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.252821:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.252827:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026b25 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.252833:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026b33 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.252837:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8bc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.252846:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8bc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.252847:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026b33 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.252847:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.252854:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026b25 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.252855:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026b2c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.252871:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.252873:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026b25 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.252875:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026b2c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.252880:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.252890:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026b33 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.252892:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026b25 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.252904:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.252911:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.252913:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026b25 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.252917:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026b33 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.252927:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503325c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+71:2390:x1870153002487296:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 223us (693us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.252936:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36597 00010000:00010000:1.0:1783520936.252936:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026b33 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.252937:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.252941:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.252945:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.252957:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026b33 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.252962:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026b17 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.252968:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509174a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+934:3628:x1870152993288576:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3729us (3960us total) trans 4294976502 rc 0/0 00010000:00010000:1.0:1783520936.252968:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026b33 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.252971:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026b17 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.252987:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45003 00010000:00010000:3.0:1783520936.252992:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026b17 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.252998:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026b17 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.253009:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.253011:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to regular blp list, 0 blwis in pool 00010000:00010000:1.0:1783520936.253014:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026b17 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.253023:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4800 already unused, calling callback (ffffffffc0bf9e90) 00000004:00080000:2.0:1783520936.253027:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943509175500/2, rc 0, transno 4294976502 00010000:00010000:1.0:1783520936.253028:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026b17 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.253040:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026b17 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.253042:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509175500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993288576:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.253048:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026b2c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.253066:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026b1e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.253069:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026b2c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.253076:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026b17 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.253077:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026b1e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.253078:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026b2c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.253100:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026b17 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.253101:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026b1e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.253109:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026b2c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.253120:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026b1e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.253125:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+934:3627:x1870152993287936:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4361us (4573us total) trans 4294976503 rc 0/0 00010000:00010000:0.0:1783520936.253133:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520936.253135:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00000100:00100000:3.0:1783520936.253138:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45000 00010000:00010000:2.0:1783520936.253140:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.253141:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026b2c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.253146:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.253153:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.253153:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026b1e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.253160:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.253160:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026b2c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.253165:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.253171:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026b1e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.253174:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026b2c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.253183:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513b04a80/2, rc 0, transno 4294976503 00010000:00010000:0.0:1783520936.253193:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026b1e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.253194:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b04a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993287936:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.253199:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026b2c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.253217:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505875f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+934:3625:x1870152993287552:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5337us (5444us total) trans 4294976504 rc 0/0 00010000:00010000:1.0:1783520936.253219:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026b2c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.253222:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026b1e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.253231:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 44998 00010000:00010000:0.0:1783520936.253242:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026b1e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8b7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.253249:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.253252:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.253253:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026b3a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.253256:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.253259:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.253262:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.253264:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.253266:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.253269:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026b3a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.253277:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026b3a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.253284:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943505876d80/2, rc 0, transno 4294976504 00010000:00010000:2.0:1783520936.253285:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026b3a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.253290:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943505876d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993287552:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.253304:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.253317:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:3.0:1783520936.253325:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520936.253330:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514dd6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993289856:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.253331:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026b3a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.253342:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.253347:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026b3a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.253381:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026b3a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.253395:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.253397:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026b3a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.253406:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351720b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+934:3625:x1870152993288192:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4003us (4505us total) trans 4294976505 rc 0/0 00010000:00010000:3.0:1783520936.253409:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026b3a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.253421:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45004 00000100:00100000:1.0:1783520936.253427:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509175880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+934:3628:x1870152993288064:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4464us (4636us total) trans 4294976501 rc 0/0 00000004:00080000:0.0:1783520936.253429:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943509177b80/2, rc 0, transno 4294976505 00000100:00100000:1.0:1783520936.253438:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45001 00010000:00010000:3.0:1783520936.253439:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.253443:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.253446:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.253447:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509177b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993288192:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.253468:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514dd7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993289984:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.253468:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b05c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993290112:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.253506:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b07100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993290240:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.253510:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993289856 00000100:00100000:1.0:1783520936.253515:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993289856 00000100:00100000:2.0:1783520936.253520:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.253542:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993290112 00000100:00100000:1.0:1783520936.253546:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993290112 00000100:00100000:0.0:1783520936.253547:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.253553:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45007 00000004:00080000:2.0:1783520936.253562:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517208000/2, rc 0, transno 4294976501 00000100:00100000:3.0:1783520936.253572:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.253572:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993290240 00000100:00100000:2.0:1783520936.253576:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517208000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993288064:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.253578:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993290240 00000100:00100000:0.0:1783520936.253593:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993289984 00000100:00100000:0.0:1783520936.253597:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993289984 00000100:00100000:1.0:1783520936.253613:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45008 00000100:00100000:3.0:1783520936.253623:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45009 00000100:00100000:1.0:1783520936.253626:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e3b1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+936:3628:x1870152993290112:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.253632:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45010 00000100:00100000:3.0:1783520936.253637:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363840df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+936:3625:x1870152993290240:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.253644:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e3b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993290368:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.253653:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032e8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+936:3627:x1870152993289984:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520936.253660:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2243 count 1 00002000:00080000:0.0:1783520936.253669:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2241 count 1 00000100:00100000:1.0:1783520936.253676:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.253677:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc026b4f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8c3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.253684:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026b56 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8c1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.253688:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000f0a4089d/0xd5401cbabc0226c9 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c70 expref: 70 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.253703:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c3:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.253707:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.253708:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d48eb5eb/0xd5401cbabc0226d0 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c54 expref: 70 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.253710:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc026b4f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.253712:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993290368 00000100:00100000:2.0:1783520936.253717:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993290368 00010000:00010000:0.0:1783520936.253730:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c1:0x0].0x0 since=38s v=(0 0) 00002000:00080000:1.0:1783520936.253733:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2242 count 1 00010000:00010000:0.0:1783520936.253734:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c1:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.253740:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45011 00010000:00010000:0.0:1783520936.253742:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026b56 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.253743:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000f0a4089d/0xd5401cbabc0226c9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c70 expref: 70 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.253749:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026b5d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8c2:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.253751:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515474700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+937:3626:x1870152993290368:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.253769:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2244 count 1 00010000:00010000:3.0:1783520936.253770:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000f0a4089d/0xd5401cbabc0226c9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c70 expref: 70 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.253773:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b8a1ab39/0xd5401cbabc0226c2 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c62 expref: 70 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.253780:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d48eb5eb/0xd5401cbabc0226d0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c54 expref: 70 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.253785:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026b64 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8c4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.253797:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032eb800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993290496:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.253798:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c2:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.253808:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.253816:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026b5d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.253818:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000589a1b3e/0xd5401cbabc0226d7 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c7e expref: 70 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.253818:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d48eb5eb/0xd5401cbabc0226d0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c54 expref: 70 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.253836:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c4:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.253839:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c4:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.253839:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363840d880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993290624:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.253843:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026b64 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.253850:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b8a1ab39/0xd5401cbabc0226c2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c62 expref: 70 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.253853:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e780 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.253866:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000589a1b3e/0xd5401cbabc0226d7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c7e expref: 70 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.253868:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b8a1ab39/0xd5401cbabc0226c2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c62 expref: 70 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.253869:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c803c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.253875:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503324700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+937:3625:x1870152993289856:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.253891:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000589a1b3e/0xd5401cbabc0226d7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c7e expref: 70 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.253899:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515474380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993290752:192.168.204.33@tcp:104: 00002000:00080000:3.0:1783520936.253906:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2240 count 1 00000100:00100000:2.0:1783520936.253911:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e3b0a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993290880:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.253922:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026b6b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8c0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.253934:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470480 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.253936:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005528da5c/0xd5401cbabc0226bb lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c46 expref: 70 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.253943:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1000 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.253946:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517208e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+938:3625:x1870152993288448:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4878us (4976us total) trans 4294976506 rc 0/0 00010000:00010000:3.0:1783520936.253948:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c0:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.253954:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c0:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.253957:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45002 00010000:00010000:3.0:1783520936.253960:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026b6b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.253963:0:35276:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000d98bf22/0xd5401cbabc0226b4 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c38 expref: 70 pid: 20415 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.253984:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514dd7100 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993289728:192.168.204.33@tcp:104: 00000004:00080000:0.0:1783520936.253986:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943509177100/2, rc 0, transno 4294976506 00010000:00010000:3.0:1783520936.253995:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005528da5c/0xd5401cbabc0226bb lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c46 expref: 69 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.253999:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509177100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993288448:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.254001:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026b48 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.254015:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026b48 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.254021:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026b48 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.254028:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005528da5c/0xd5401cbabc0226bb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c46 expref: 69 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.254031:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026b48 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.254047:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026b48 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.254055:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993291008:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.254056:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1783520936.254068:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94363840e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993291136:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.254075:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.254087:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.254092:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026b48 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.254100:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbaa00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.254104:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026b48 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.254111:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.254113:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026b48 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.254126:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026b48 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.254138:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993291136 00010000:00010000:0.0:1783520936.254139:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026b48 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.254142:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993291136 00010000:00010000:0.0:1783520936.254156:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.254159:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.254160:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e840 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.254162:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.254169:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351664b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+939:3625:x1870152993289600:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3342us (3414us total) trans 4294976507 rc 0/0 00000100:00100000:2.0:1783520936.254172:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45012 00000100:00100000:1.0:1783520936.254182:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45006 00000100:00100000:2.0:1783520936.254186:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363573b850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+939:3628:x1870152993291136:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.254200:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2245 count 1 00010000:00010000:2.0:1783520936.254211:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026b72 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8c5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.254217:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514dd6680/2, rc 0, transno 4294976507 00010000:00010000:2.0:1783520936.254225:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000153ab2fd/0xd5401cbabc0226de lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c8c expref: 69 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.254226:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514dd6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993289600:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.254238:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c5:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.254242:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.254246:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026b72 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.254256:0:8410:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000000efada50/0xd5401cbabc0228f9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->8392703) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2fe9 expref: 69 pid: 8409 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.254266:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000153ab2fd/0xd5401cbabc0226de lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c8c expref: 69 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.254269:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b23480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993289472:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.254285:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000153ab2fd/0xd5401cbabc0226de lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c8c expref: 68 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.254289:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026b41 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.254304:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351664bb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993291392:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.254305:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026b41 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.254315:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026b41 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.254323:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026b41 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.254325:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:1.0:1783520936.254333:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026b41 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.254336:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517333b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993291264:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.254338:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1cc0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.254417:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.254423:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026b41 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.254428:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to regular blp list, 0 blwis in pool 00010000:00010000:0.0:1783520936.254432:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.254437:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026b41 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.254443:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026b41 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.254452:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.254452:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026b41 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.254460:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026b41 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.254477:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.254480:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.254484:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.254499:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993291264 00000100:00100000:2.0:1783520936.254504:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993291264 00000100:00100000:1.0:1783520936.254528:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+940:3627:x1870152993289088:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4935us (5018us total) trans 4294976508 rc 0/0 00000100:00100000:2.0:1783520936.254534:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45013 00000004:00080000:3.0:1783520936.254537:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943509176a00/2, rc 0, transno 4294976508 00000100:00100000:1.0:1783520936.254541:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45005 00000100:00100000:2.0:1783520936.254545:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507acf100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+940:3627:x1870152993291264:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.254558:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509176a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993289088:0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.254559:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2246 count 1 00010000:00010000:2.0:1783520936.254573:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026b79 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8c6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.254589:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000034360f5d/0xd5401cbabc022b53 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3243 expref: 68 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.254596:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x8c6:0x0].0x0 since=37s v=(0 0) 00010000:00010000:2.0:1783520936.254599:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.254602:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026b79 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.254620:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000034360f5d/0xd5401cbabc022b53 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3243 expref: 68 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.254624:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:2.0:1783520936.254634:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000034360f5d/0xd5401cbabc022b53 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3243 expref: 68 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.254636:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517332d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993291520:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.254650:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351664aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993291648:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.254670:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.254695:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993291520 00000100:00100000:2.0:1783520936.254697:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1780 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.254701:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993291520 00000100:00100000:3.0:1783520936.254739:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45014 00000100:00100000:3.0:1783520936.254748:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515476680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+940:3626:x1870152993291520:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520936.254761:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2247 count 1 00010000:00010000:3.0:1783520936.254772:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026b80 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8c7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.254784:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000099120554/0xd5401cbabc0226ec lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ca8 expref: 68 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.254830:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c7:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.254833:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.254835:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026b80 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.254850:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000099120554/0xd5401cbabc0226ec lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ca8 expref: 68 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.254856:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000099120554/0xd5401cbabc0226ec lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ca8 expref: 68 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.254863:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc0000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993291776:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.254890:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.256109:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.256140:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515474380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993290752:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.256154:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032eb800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993290496:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.256159:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026b5d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.256165:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.256173:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc026b4f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.256188:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002487424 00000100:00100000:2.0:1783520936.256198:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002487424 00010000:00010000:2.0:1783520936.256222:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000f0a4089d/0xd5401cbabc0226c9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c70 expref: 69 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.256245:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002487552 00000100:00100000:2.0:1783520936.256248:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002487552 00000100:00100000:1.0:1783520936.256259:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36598 00010000:00010000:2.0:1783520936.256266:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b8a1ab39/0xd5401cbabc0226c2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c62 expref: 70 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.256268:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.256275:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517333480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+70:2388:x1870153002487424:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.256291:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.256294:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e3b0a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993290880:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.256301:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000f0a4089d/0xd5401cbabc0226c9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8c3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c70 expref: 70 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.256307:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002487680 00000100:00100000:2.0:1783520936.256312:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002487680 00010000:00010000:3.0:1783520936.256314:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026b64 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.256320:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000f0a4089d/0xd5401cbabc0226c9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8c3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c70 expref: 71 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.256331:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d48eb5eb/0xd5401cbabc0226d0 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c54 expref: 71 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.256337:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.256346:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993291008:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.256387:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f0a4089d/0xd5401cbabc0226c9 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8c3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c70 expref: 71 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.256399:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002487808 00000100:00100000:2.0:1783520936.256405:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002487808 00000100:00100000:0.0:1783520936.256405:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.256418:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.256422:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.256426:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000589a1b3e/0xd5401cbabc0226d7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c7e expref: 71 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.256429:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc026b4f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.256432:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026b6b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.256453:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc026b4f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.256453:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.256456:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002487936 00000100:00100000:2.0:1783520936.256461:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002487936 00010000:00010000:2.0:1783520936.256481:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005528da5c/0xd5401cbabc0226bb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8c0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c46 expref: 72 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.256482:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.256512:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002488064 00000100:00100000:2.0:1783520936.256518:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36599 00000100:00100000:0.0:1783520936.256519:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002488064 00000100:00100000:1.0:1783520936.256529:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517333480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+73:2388:x1870153002487424:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 258us (420us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.256533:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943634573b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+73:2387:x1870153002487552:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.256537:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc026b4f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.256544:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36598 00010000:00010000:0.0:1783520936.256544:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000153ab2fd/0xd5401cbabc0226de lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c8c expref: 73 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.256547:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.256558:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b8a1ab39/0xd5401cbabc0226c2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8c2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c62 expref: 73 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.256560:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc026b4f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.256570:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36600 00000100:00100000:1.0:1783520936.256584:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+72:2388:x1870153002487680:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.256585:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc026b4f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.256588:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b8a1ab39/0xd5401cbabc0226c2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8c2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c62 expref: 72 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.256595:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36601 00010000:00010000:1.0:1783520936.256599:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.256609:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc026b4f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.256610:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d48eb5eb/0xd5401cbabc0226d0 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8c1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c54 expref: 72 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.256612:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+72:2389:x1870153002487808:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.256625:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b8a1ab39/0xd5401cbabc0226c2 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8c2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c62 expref: 72 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.256627:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.256633:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d48eb5eb/0xd5401cbabc0226d0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8c1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c54 expref: 72 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.256638:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000589a1b3e/0xd5401cbabc0226d7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8c4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c7e expref: 72 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.256648:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.256658:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.256660:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.256668:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.256670:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026b5d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.256671:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000589a1b3e/0xd5401cbabc0226d7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8c4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c7e expref: 71 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.256673:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026b56 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.256681:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.256698:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026b56 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8c1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.256701:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026b5d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.256707:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000589a1b3e/0xd5401cbabc0226d7 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8c4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c7e expref: 71 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.256715:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.256722:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc026b4f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.256730:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.256731:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.256733:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc026b4f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.256737:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.256739:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.256741:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc026b4f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.256743:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026b64 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.256752:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc026b4f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.256753:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026b64 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.256764:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943634573b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+70:2387:x1870153002487552:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 233us (600us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.256767:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c55d0c3e/0xd5401cbabc026b4f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.256780:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36599 00000100:00100000:3.0:1783520936.256786:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363840df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+941:3625:x1870152993290240:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3146us (3233us total) trans 4294976509 rc 0/0 00010000:00010000:0.0:1783520936.256790:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.256792:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.256796:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.256800:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.256803:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45009 00000100:00100000:2.0:1783520936.256805:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36602 00000100:00100000:2.0:1783520936.256822:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f3050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+69:2389:x1870153002487936:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.256822:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+69:2389:x1870153002487808:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 210us (485us total) trans 0 rc 0/0 00000004:00080000:1.0:1783520936.256824:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513b07100/2, rc 0, transno 4294976509 00010000:00010000:3.0:1783520936.256827:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026b64 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.256838:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.256843:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026b64 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.256844:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36601 00000100:00100000:1.0:1783520936.256846:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b07100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993290240:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.256848:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005528da5c/0xd5401cbabc0226bb lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8c0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c46 expref: 69 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.256872:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026b64 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.256874:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005528da5c/0xd5401cbabc0226bb lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8c0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c46 expref: 68 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.256886:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36603 00010000:00010000:3.0:1783520936.256897:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026b64 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.256904:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005528da5c/0xd5401cbabc0226bb lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8c0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c46 expref: 68 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.256904:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435166cbb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+68:2387:x1870153002488064:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.256906:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435166c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993291904:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.256921:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.256927:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.256932:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.256932:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000153ab2fd/0xd5401cbabc0226de lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8c5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c8c expref: 67 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.256940:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.256946:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.256948:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026b6b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.256949:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.256955:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026b64 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.256961:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026b6b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.256961:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000153ab2fd/0xd5401cbabc0226de lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8c5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c8c expref: 67 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.256971:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.256978:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026b64 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.256991:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026b64 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.256992:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.256993:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.256999:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.257008:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026b72 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.257016:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026b64 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.257016:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026b72 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8c5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.257027:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f3050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+67:2389:x1870153002487936:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 210us (623us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.257029:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.257036:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.257042:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026b64 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.257049:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36602 00000100:00100000:1.0:1783520936.257057:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435166cbb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+67:2387:x1870153002488064:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 159us (605us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.257065:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36603 00010000:00010000:3.0:1783520936.257072:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.257072:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002488192 00010000:00010000:3.0:1783520936.257077:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.257079:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002488192 00010000:00010000:3.0:1783520936.257082:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.257091:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.257098:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000034360f5d/0xd5401cbabc022b53 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x8c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3243 expref: 66 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.257117:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+66:2388:x1870153002487680:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 534us (851us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.257125:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36604 00000100:00100000:2.0:1783520936.257139:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435166cb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+66:2387:x1870153002488192:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.257139:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36600 00010000:00010000:2.0:1783520936.257152:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.257159:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000034360f5d/0xd5401cbabc022b53 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x8c6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3243 expref: 65 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.257171:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.257174:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000034360f5d/0xd5401cbabc022b53 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8c6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3243 expref: 65 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.257181:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002488320 00000100:00100000:1.0:1783520936.257186:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002488320 00000100:00100000:3.0:1783520936.257190:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515474700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+941:3626:x1870152993290368:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3442us (3514us total) trans 4294976510 rc 0/0 00010000:00010000:2.0:1783520936.257191:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.257199:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000099120554/0xd5401cbabc0226ec lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ca8 expref: 66 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.257200:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45011 00010000:00010000:2.0:1783520936.257201:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c6:0x0].0x0 since=0s v=(1 0) 00000004:00080000:0.0:1783520936.257206:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e3b0700/2, rc 0, transno 4294976510 00010000:00010000:2.0:1783520936.257212:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026b79 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.257218:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993291904 00000100:00100000:0.0:1783520936.257223:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e3b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993290368:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.257224:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026b79 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8c6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.257225:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993291904 00000100:00100000:1.0:1783520936.257226:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36605 00000100:00100000:1.0:1783520936.257238:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435166c8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+66:2389:x1870153002488320:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.257246:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.257248:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.257250:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000099120554/0xd5401cbabc0226ec lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8c7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ca8 expref: 66 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.257262:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000099120554/0xd5401cbabc0226ec lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8c7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ca8 expref: 66 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.257267:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45015 00000100:00100000:2.0:1783520936.257275:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435166cb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+66:2387:x1870153002488192:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 141us (252us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.257277:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.257284:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c7:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.257285:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435166c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993292032:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.257289:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026b80 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.257291:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36604 00010000:00010000:1.0:1783520936.257300:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026b80 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8c7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.257310:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.257327:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435166c8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+941:3626:x1870152993291904:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.257327:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.257331:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435166c8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+65:2389:x1870153002488320:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 92us (162us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.257340:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36605 00002000:00080000:3.0:1783520936.257349:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2248 count 1 00010000:00010000:2.0:1783520936.257349:0:9151:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000034360f5d/0xd5401cbabc022b53 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 54525952->54530047) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3243 expref: 64 pid: 35275 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.257383:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351664aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993291648:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.257387:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026b5d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.257390:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026b87 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8c8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.257400:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026b79 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.257410:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026b5d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.257420:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000cfb28e6d/0xd5401cbabc0226f3 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cb6 expref: 63 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.257425:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026b79 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.257434:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026b5d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.257446:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c8:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.257449:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026b79 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.257456:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.257459:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026b5d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.257466:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026b87 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.257503:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026b79 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.257505:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.257511:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.257519:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000cfb28e6d/0xd5401cbabc0226f3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cb6 expref: 63 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.257520:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026b5d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.257521:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026b79 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.257548:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.257551:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000cfb28e6d/0xd5401cbabc0226f3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cb6 expref: 63 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.257554:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026b5d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.257556:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.257573:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026b5d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.257578:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e3b1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+942:3628:x1870152993290112:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3956us (4056us total) trans 4294976511 rc 0/0 00000100:00100000:1.0:1783520936.257592:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45008 00010000:00010000:0.0:1783520936.257595:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026b5d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.257604:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026b5d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.257610:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993292032 00000100:00100000:1.0:1783520936.257616:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993292032 00000100:00100000:3.0:1783520936.257617:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993292160:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.257629:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.257634:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026b79 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.257642:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.257647:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026b79 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.257650:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45016 00010000:00010000:0.0:1783520936.257660:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026b79 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.257662:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435166c8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+943:3627:x1870152993292032:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.257666:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507acf100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+943:3627:x1870152993291264:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3122us (3212us total) trans 4294976512 rc 0/0 00000100:00100000:3.0:1783520936.257676:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e780 going to sleep for 11 seconds 00002000:00080000:1.0:1783520936.257681:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2249 count 1 00000100:00100000:2.0:1783520936.257682:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45013 00010000:00010000:0.0:1783520936.257688:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026b79 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.257695:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026b8e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8c9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.257698:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026b79 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.257710:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513b05c00/2, rc 0, transno 4294976511 00010000:00010000:1.0:1783520936.257711:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003ce50287/0xd5401cbabc0226fa lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cc4 expref: 63 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.257720:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.257721:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c9:0x0].0x0 since=38s v=(0 0) 00000100:00100000:3.0:1783520936.257724:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b05c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993290112:0@lo:6:osp-syn-1-0.0 00000004:00080000:2.0:1783520936.257725:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517333b80/2, rc 0, transno 4294976512 00010000:00010000:0.0:1783520936.257725:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.257727:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.257731:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.257737:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517333b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993291264:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.257738:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026b8e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.257741:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.257749:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026b6b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.257756:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026b6b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.257763:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026b6b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.257767:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003ce50287/0xd5401cbabc0226fa lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cc4 expref: 63 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.257778:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026b6b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.257778:0:35282:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000153ab2fd/0xd5401cbabc0226de lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c8c expref: 63 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.257789:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003ce50287/0xd5401cbabc0226fa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cc4 expref: 63 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.257794:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1783520936.257807:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9436342b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993292544:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.257813:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351664bb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993291392:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.257817:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515474380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993292288:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.257819:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.257832:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026b72 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.257833:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.257841:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026b6b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.257845:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.257854:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026b72 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.257856:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.257864:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026b6b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.257866:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026b72 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.257881:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:1.0:1783520936.257885:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026b6b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.257892:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943508ee4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993292416:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.257901:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026b72 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.257905:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503324700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+943:3625:x1870152993289856:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4341us (4509us total) trans 4294976513 rc 0/0 00010000:00010000:1.0:1783520936.257910:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026b6b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.257916:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026b72 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.257922:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.257925:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45007 00010000:00010000:1.0:1783520936.257936:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009d2fa220/0xd5401cbabc026b6b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.257944:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993292544 00010000:00010000:2.0:1783520936.257946:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.257948:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993292544 00010000:00010000:3.0:1783520936.257951:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.257953:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.257956:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.257958:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026b72 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.257958:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.257966:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993292416 00010000:00010000:3.0:1783520936.257969:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185600 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520936.257970:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993292416 00010000:00010000:3.0:1783520936.257976:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026b72 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.257984:0:8409:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d48eb5eb/0xd5401cbabc0226d0 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2c54 expref: 62 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.257987:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026b72 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.257987:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45017 00000100:00100000:0.0:1783520936.258000:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435038ed180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+945:3626:x1870152993292544:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.258006:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363840d880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993290624:192.168.204.33@tcp:104: 00002000:00080000:0.0:1783520936.258017:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2251 count 1 00010000:00010000:3.0:1783520936.258019:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026b72 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.258025:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026b56 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.258031:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000841b1b91/0xd5401cbabc026b72 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.258033:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363573b850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+945:3628:x1870152993291136:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3847us (3918us total) trans 4294976514 rc 0/0 00010000:00010000:0.0:1783520936.258034:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026b95 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8cb:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.258047:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026b56 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.258054:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45012 00010000:00010000:1.0:1783520936.258064:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026b56 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.258068:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000627806c4/0xd5401cbabc022701 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ce0 expref: 61 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.258072:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.258073:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026b56 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.258077:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.258081:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45018 00010000:00010000:3.0:1783520936.258082:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.258087:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8cb:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.258090:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026b56 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.258092:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8cb:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.258097:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517244e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+944:3627:x1870152993292416:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.258103:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026b95 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.258108:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514dd6d80/2, rc 0, transno 4294976513 00002000:00080000:2.0:1783520936.258116:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2250 count 1 00010000:00010000:1.0:1783520936.258124:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.258131:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026b9c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8ca:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.258132:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514dd6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993289856:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.258133:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.258139:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026b56 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.258153:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.258156:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ddf4bc6f/0xd5401cbabc022708 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cd2 expref: 61 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.258157:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026b56 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.258165:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94363840e680/2, rc 0, transno 4294976514 00010000:00010000:2.0:1783520936.258172:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ca:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.258175:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026b56 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.258176:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94363840e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993291136:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.258176:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ca:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.258184:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026b9c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.258193:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032e8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+945:3627:x1870152993289984:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4551us (4620us total) trans 4294976515 rc 0/0 00010000:00010000:0.0:1783520936.258197:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026b56 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.258212:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ddf4bc6f/0xd5401cbabc022708 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cd2 expref: 61 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.258213:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026b56 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.258215:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45010 00010000:00010000:3.0:1783520936.258218:0:20140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000099120554/0xd5401cbabc0226ec lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ca8 expref: 61 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.258246:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ddf4bc6f/0xd5401cbabc022708 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cd2 expref: 61 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.258254:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.258257:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc0000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993291776:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.258259:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.258263:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351664bb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993292672:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.258265:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.258279:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026b80 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.258292:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026b80 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.258297:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1900 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.258300:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026b80 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.258312:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026b80 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.258321:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026b80 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.258325:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516312a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993292800:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.258340:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1783520936.258351:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516310000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993292928:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.258409:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.258431:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993292800 00000100:00100000:2.0:1783520936.258437:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993292800 00000100:00100000:3.0:1783520936.258442:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.258443:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470f00 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.258466:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993292928 00000100:00100000:2.0:1783520936.258469:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993292928 00000004:00080000:0.0:1783520936.258474:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514dd7800/2, rc 0, transno 4294976515 00000100:00100000:3.0:1783520936.258484:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45019 00000100:00100000:0.0:1783520936.258485:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514dd7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993289984:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.258507:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45020 00000100:00100000:3.0:1783520936.258513:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ae5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+946:3626:x1870152993292800:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.258521:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513bc0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+946:3628:x1870152993292928:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520936.258528:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2252 count 1 00002000:00080000:2.0:1783520936.258537:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2253 count 1 00010000:00010000:3.0:1783520936.258543:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026ba3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8cc:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.258552:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026baa lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8cd:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.258558:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000052d47488/0xd5401cbabc02270f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cee expref: 60 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.258567:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:0.0:1783520936.258569:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000627806c4/0xd5401cbabc022701 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ce0 expref: 60 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.258570:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b4251a78/0xd5401cbabc022716 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cfc expref: 60 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.258572:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8cc:0x0].0x0 since=38s v=(0 0) 00000100:00100000:1.0:1783520936.258580:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e472300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993293056:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.258582:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8cc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.258594:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8cd:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.258595:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026ba3 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.258600:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8cd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.258601:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000627806c4/0xd5401cbabc022701 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ce0 expref: 60 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.258608:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026baa lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.258620:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.258632:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000052d47488/0xd5401cbabc02270f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cee expref: 60 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.258637:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350793ad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993293184:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.258645:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b4251a78/0xd5401cbabc022716 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cfc expref: 60 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.258650:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993293056 00010000:00010000:3.0:1783520936.258654:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000052d47488/0xd5401cbabc02270f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cee expref: 60 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.258656:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993293056 00010000:00010000:2.0:1783520936.258673:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b4251a78/0xd5401cbabc022716 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cfc expref: 60 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.258673:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80600 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.258677:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e3b1880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993293312:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.258688:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45021 00000100:00100000:2.0:1783520936.258700:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351664aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993293440:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.258701:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515474700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+947:3627:x1870152993293056:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.258714:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00002000:00080000:1.0:1783520936.258719:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2254 count 1 00010000:00010000:1.0:1783520936.258734:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026bb1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8ce:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.258746:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1a80 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.258747:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ab8c5b1e/0xd5401cbabc022a42 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3132 expref: 60 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.258756:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x8ce:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.258760:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ce:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.258764:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026bb1 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.258773:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.258784:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ab8c5b1e/0xd5401cbabc022a42 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3132 expref: 60 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.258794:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ab8c5b1e/0xd5401cbabc022a42 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3132 expref: 60 pid: 12734 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.258806:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1783520936.258811:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515476a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993293568:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.258814:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026b80 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.258824:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.258828:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026b80 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.258838:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026b80 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.258850:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470a80 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.258852:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026b80 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.258862:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b5f9e918/0xd5401cbabc026b80 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.258865:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515476680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+948:3626:x1870152993291520:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4117us (4194us total) trans 4294976516 rc 0/0 00000100:00100000:2.0:1783520936.258881:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45014 00010000:00010000:0.0:1783520936.258890:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.258893:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.258896:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520936.258920:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943517332d80/2, rc 0, transno 4294976516 00000100:00100000:0.0:1783520936.258943:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517332d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993291520:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.259034:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351cc7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993293696:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.259064:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.259107:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993293696 00000100:00100000:1.0:1783520936.259112:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993293696 00000100:00100000:1.0:1783520936.259135:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45022 00000100:00100000:1.0:1783520936.259142:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350793bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+948:3625:x1870152993293696:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.259153:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2255 count 1 00010000:00010000:1.0:1783520936.259164:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026bb8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8cf:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.259174:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000032b19313/0xd5401cbabc022724 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d18 expref: 60 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.259180:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8cf:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.259194:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8cf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.259198:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026bb8 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.259216:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000032b19313/0xd5401cbabc022724 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d18 expref: 60 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.259224:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000032b19313/0xd5401cbabc022724 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d18 expref: 60 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.259239:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9ad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993293824:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.259269:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470000 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.260575:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993292160:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.260597:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026b87 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.260605:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351664bb80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993292672:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.260628:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026b9c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.260634:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515474380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993292288:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.260658:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026b8e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.260680:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e3b1880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993293312:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.260698:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026ba3 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.260703:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515476a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993293568:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.260715:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9ad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993293824:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.260724:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350793ad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993293184:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.260725:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026bb1 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.260736:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026bb8 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.260746:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026b95 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.260776:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351664aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993293440:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.260786:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026baa lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.260879:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.260936:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002488832 00000100:00100000:2.0:1783520936.260941:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002488832 00010000:00010000:2.0:1783520936.260961:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b4251a78/0xd5401cbabc022716 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cfc expref: 61 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.260986:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36606 00000100:00100000:2.0:1783520936.260994:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e3b1880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+61:2390:x1870153002488832:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.261002:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.261008:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b4251a78/0xd5401cbabc022716 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8cd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cfc expref: 61 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.261019:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b4251a78/0xd5401cbabc022716 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8cd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cfc expref: 61 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.261037:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b4251a78/0xd5401cbabc022716 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8cd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cfc expref: 61 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.261085:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8cd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.261090:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8cd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.261096:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026baa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.261105:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026baa lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.261129:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.261144:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.261158:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e3b1880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+60:2390:x1870153002488832:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 165us (279us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.261164:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36606 00010000:00010000:1.0:1783520936.261168:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026baa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.261178:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002489088 00010000:00010000:1.0:1783520936.261179:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026baa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.261182:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002489088 00010000:00010000:1.0:1783520936.261192:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026baa lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.261195:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ab8c5b1e/0xd5401cbabc022a42 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3132 expref: 60 pid: 12734 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.261212:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026baa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.261223:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36607 00000100:00100000:2.0:1783520936.261229:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350793ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+60:2389:x1870153002489088:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.261233:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.261236:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.261239:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.261242:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ab8c5b1e/0xd5401cbabc022a42 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8ce:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3132 expref: 60 pid: 12734 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.261248:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.261252:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ab8c5b1e/0xd5401cbabc022a42 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8ce:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3132 expref: 60 pid: 12734 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.261256:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026baa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.261271:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185a00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520936.261273:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.261277:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026baa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.261289:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026baa lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.261290:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ab8c5b1e/0xd5401cbabc022a42 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8ce:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 33554432->33558527) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3132 expref: 60 pid: 12734 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.261295:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002488448 00000100:00100000:3.0:1783520936.261302:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002488448 00000100:00100000:0.0:1783520936.261307:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.261318:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ce:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.261318:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026baa lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.261325:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000cfb28e6d/0xd5401cbabc0226f3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cb6 expref: 60 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.261325:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ce:0x0].0x0 since=0s v=(1 0) 00000100:00100000:0.0:1783520936.261337:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.261347:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026bb1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.261348:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000354fa27c/0xd5401cbabc026baa lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.261390:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026bb1 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.261398:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.261401:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.261404:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.261410:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.261418:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002488704 00000100:00100000:3.0:1783520936.261422:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002489216 00000100:00100000:1.0:1783520936.261423:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002488704 00000100:00100000:3.0:1783520936.261429:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002489216 00000100:00100000:2.0:1783520936.261441:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350793ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+62:2389:x1870153002489088:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 210us (297us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.261446:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003ce50287/0xd5401cbabc0226fa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cc4 expref: 62 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.261451:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000032b19313/0xd5401cbabc022724 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d18 expref: 62 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.261455:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36607 00000100:00100000:0.0:1783520936.261474:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002488576 00000100:00100000:2.0:1783520936.261478:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36608 00000100:00100000:0.0:1783520936.261479:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002488576 00000100:00100000:1.0:1783520936.261504:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36609 00000100:00100000:2.0:1783520936.261505:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350626dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+62:2387:x1870153002488448:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.261512:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ddf4bc6f/0xd5401cbabc022708 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cd2 expref: 62 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.261518:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.261520:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f3450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+62:2387:x1870153002488704:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.261527:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000cfb28e6d/0xd5401cbabc0226f3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8c8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cb6 expref: 62 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.261531:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36610 00010000:00010000:1.0:1783520936.261543:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.261548:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350626e680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+62:2387:x1870153002489216:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.261550:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000cfb28e6d/0xd5401cbabc0226f3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8c8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cb6 expref: 62 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.261551:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36611 00010000:00010000:1.0:1783520936.261554:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003ce50287/0xd5401cbabc0226fa lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8c9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cc4 expref: 62 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.261565:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.261565:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350626ed80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+62:2387:x1870153002488576:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.261575:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000032b19313/0xd5401cbabc022724 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8cf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d18 expref: 62 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.261580:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.261584:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003ce50287/0xd5401cbabc0226fa lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8c9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cc4 expref: 62 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.261585:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cfb28e6d/0xd5401cbabc0226f3 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8c8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cb6 expref: 62 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.261589:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ddf4bc6f/0xd5401cbabc022708 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ca:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cd2 expref: 62 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.261595:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000032b19313/0xd5401cbabc022724 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8cf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d18 expref: 62 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.261618:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.261626:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003ce50287/0xd5401cbabc0226fa lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8c9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cc4 expref: 61 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.261627:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.261633:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000032b19313/0xd5401cbabc022724 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8cf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d18 expref: 61 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.261633:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ddf4bc6f/0xd5401cbabc022708 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8ca:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cd2 expref: 61 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.261637:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026b87 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.261655:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.261664:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8cf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.261665:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026b87 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.261665:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8c9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.261668:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ddf4bc6f/0xd5401cbabc022708 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8ca:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cd2 expref: 59 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.261673:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8cf:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.261676:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026b8e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.261684:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026bb8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.261700:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.261702:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026bb8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.261702:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ca:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.261708:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026b8e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.261710:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ca:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.261723:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350626dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+58:2387:x1870153002488448:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 232us (485us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.261723:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026b9c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.261739:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026b9c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.261740:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.261740:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36608 00010000:00010000:1.0:1783520936.261747:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.261761:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.261769:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350626e680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+57:2387:x1870153002489216:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 226us (467us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.261771:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026bb1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.261779:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f3450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+57:2387:x1870153002488704:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 262us (506us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.261787:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36610 00000100:00100000:0.0:1783520936.261787:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350626ed80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+57:2387:x1870153002488576:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 221us (449us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.261800:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36609 00010000:00010000:2.0:1783520936.261806:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026bb1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.261811:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36611 00010000:00010000:3.0:1783520936.261826:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026b87 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.261836:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026b87 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.261845:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026b87 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.261854:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.261859:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026b87 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.261877:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.261878:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026bb1 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.261879:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002488960 00000100:00100000:1.0:1783520936.261885:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002488960 00010000:00010000:3.0:1783520936.261887:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520936.261889:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.261896:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026b87 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.261898:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026bb1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.261906:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000627806c4/0xd5401cbabc022701 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ce0 expref: 55 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.261928:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.261932:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.261939:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026b87 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.261939:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.261948:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026bb1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.261952:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026b87 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.261953:0:6479:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002489344 00000100:00100000:1.0:1783520936.261957:0:6479:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002489344 00010000:00010000:0.0:1783520936.261967:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbbe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.261973:0:6479:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000052d47488/0xd5401cbabc02270f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cee expref: 56 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.261975:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026bb1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.261976:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026b87 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.262002:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026b87 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.262004:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026bb1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.262006:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36612 00000100:00100000:1.0:1783520936.262020:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350626d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+56:2387:x1870153002488960:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.262035:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.262035:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.262036:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026bb1 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.262039:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.262044:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.262045:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000627806c4/0xd5401cbabc022701 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8cb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ce0 expref: 56 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.262049:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515474700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+949:3627:x1870152993293056:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3348us (3430us total) trans 4294976517 rc 0/0 00010000:00010000:0.0:1783520936.262054:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026bb1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.262063:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36613 00000100:00100000:2.0:1783520936.262068:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45021 00010000:00010000:1.0:1783520936.262072:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000627806c4/0xd5401cbabc022701 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8cb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ce0 expref: 56 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.262078:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350626dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+56:2387:x1870153002489344:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.262085:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.262088:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.262091:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.262094:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.262096:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000627806c4/0xd5401cbabc022701 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8cb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ce0 expref: 56 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.262105:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000052d47488/0xd5401cbabc02270f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8cc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cee expref: 56 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.262105:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026b9c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.262109:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8cb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.262118:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8cb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.262127:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026b95 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.262129:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026b9c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.262136:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000052d47488/0xd5401cbabc02270f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8cc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cee expref: 55 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.262145:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026b95 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8cb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.262155:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026b9c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.262179:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026b9c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.262188:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000052d47488/0xd5401cbabc02270f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8cc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2cee expref: 55 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.262201:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.262213:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8cc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.262218:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8cc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.262218:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.262223:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.262226:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026ba3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.262227:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350626d180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+54:2387:x1870153002488960:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 209us (376us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.262231:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026b9c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.262245:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36612 00010000:00010000:3.0:1783520936.262251:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026ba3 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8cc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.262255:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.262261:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026b9c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.262272:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026b9c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.262275:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000004:00080000:1.0:1783520936.262290:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e472300/2, rc 0, transno 4294976517 00010000:00010000:0.0:1783520936.262290:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026b9c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.262302:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517244e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+949:3627:x1870152993292416:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4209us (4381us total) trans 4294976518 rc 0/0 00000100:00100000:3.0:1783520936.262305:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350626dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+53:2387:x1870153002489344:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 230us (417us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.262307:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e472300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993293056:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.262308:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003a22489f/0xd5401cbabc026b9c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.262320:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45018 00000100:00100000:3.0:1783520936.262326:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36613 00000004:00080000:1.0:1783520936.262330:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943508ee4380/2, rc 0, transno 4294976518 00000100:00100000:1.0:1783520936.262337:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943508ee4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993292416:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.262343:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.262347:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.262350:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.262423:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026b95 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.262432:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514e9ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993293952:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.262434:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026b95 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.262447:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026b95 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.262456:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026b95 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.262462:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435166cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993294080:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.262472:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.262505:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.262516:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.262544:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993293952 00000100:00100000:2.0:1783520936.262549:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993293952 00010000:00010000:1.0:1783520936.262549:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.262557:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026b95 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.262565:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbebe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.262569:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026b95 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.262575:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993294080 00010000:00010000:1.0:1783520936.262578:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026b95 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.262579:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993294080 00010000:00010000:1.0:1783520936.262597:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026b95 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.262598:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435038ed180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+951:3626:x1870152993292544:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4600us (4756us total) trans 4294976519 rc 0/0 00010000:00010000:3.0:1783520936.262604:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026ba3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.262607:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45023 00000100:00100000:0.0:1783520936.262619:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45017 00000100:00100000:2.0:1783520936.262621:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351664bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+951:3626:x1870152993293952:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.262633:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026ba3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.262635:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002ba16504/0xd5401cbabc026b95 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.262643:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2256 count 1 00000004:00080000:0.0:1783520936.262650:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9436342b8000/2, rc 0, transno 4294976519 00010000:00010000:3.0:1783520936.262656:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026ba3 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.262661:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026bbf lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8d0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.262665:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.262665:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9436342b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993292544:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.262670:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026ba3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.262670:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.262675:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.262687:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008163e037/0xd5401cbabc02272b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d26 expref: 52 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.262695:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.262697:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d0:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.262701:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d0:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.262706:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45024 00010000:00010000:3.0:1783520936.262707:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.262707:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026bbf lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.262715:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026ba3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.262720:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435166cb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+950:3627:x1870152993294080:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.262731:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.262734:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008163e037/0xd5401cbabc02272b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d26 expref: 52 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.262737:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026ba3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.262739:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2257 count 1 00010000:00010000:1.0:1783520936.262755:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026bc6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8d1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.262756:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026ba3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.262758:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008163e037/0xd5401cbabc02272b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d26 expref: 52 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.262787:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000005fa00a3d/0xd5401cbabc022732 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d34 expref: 52 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.262788:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026ba3 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.262788:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516649500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993294336:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.262810:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d1:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.262812:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026ba3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.262816:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.262825:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026bc6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.262836:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.262839:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.262841:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.262850:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000005fa00a3d/0xd5401cbabc022732 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d34 expref: 52 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.262854:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993294208:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.262868:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000005fa00a3d/0xd5401cbabc022732 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d34 expref: 52 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.262889:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b16c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.262889:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435166c8000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993294464:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.262898:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.262919:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993294208 00000100:00100000:2.0:1783520936.262924:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993294208 00000100:00100000:1.0:1783520936.262926:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470f00 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.262955:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45025 00010000:00010000:1.0:1783520936.262962:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026b8e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.262967:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e3b0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+951:3628:x1870152993294208:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.262978:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026b8e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.262984:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2258 count 1 00010000:00010000:1.0:1783520936.262988:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026b8e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.262997:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026bcd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8d2:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.262998:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026b8e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.263024:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002c725817/0xd5401cbabc022739 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d42 expref: 52 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.263028:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.263036:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d2:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.263038:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.263040:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.263047:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026b8e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.263048:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026bcd lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.263070:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.263075:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026b8e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.263084:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002c725817/0xd5401cbabc022739 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d42 expref: 52 pid: 12734 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.263086:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026b8e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.263113:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026b8e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.263114:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002c725817/0xd5401cbabc022739 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d42 expref: 52 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.263119:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435166c8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+952:3627:x1870152993292032:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5457us (5794us total) trans 4294976520 rc 0/0 00000100:00100000:1.0:1783520936.263136:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45016 00010000:00010000:0.0:1783520936.263137:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b4783b77/0xd5401cbabc026b8e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.263141:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351664aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993294592:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.263164:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.263166:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.263169:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.263177:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b19c0 going to sleep for 11 seconds 00000004:00080000:1.0:1783520936.263178:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435166c8a80/2, rc 0, transno 4294976520 00000100:00100000:1.0:1783520936.263197:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435166c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993292032:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.263242:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435166c8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+952:3626:x1870152993291904:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5921us (6297us total) trans 4294976521 rc 0/0 00000100:00100000:3.0:1783520936.263256:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45015 00000004:00080000:2.0:1783520936.263259:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435166c9c00/2, rc 0, transno 4294976521 00000100:00100000:2.0:1783520936.263271:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435166c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993291904:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.263273:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351530dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993294720:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.263299:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513bc0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+952:3628:x1870152993292928:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4782us (4860us total) trans 4294976522 rc 0/0 00000100:00100000:1.0:1783520936.263310:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45020 00000100:00100000:0.0:1783520936.263311:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.263331:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993294720 00000100:00100000:1.0:1783520936.263336:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993294720 00000100:00100000:1.0:1783520936.263404:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45026 00000100:00100000:2.0:1783520936.263412:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507ad3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993294848:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.263416:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ad2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+952:3625:x1870152993294720:12345-0@lo:6:osp-syn-1-0.0 00000004:00080000:3.0:1783520936.263423:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516310000/2, rc 0, transno 4294976522 00002000:00080000:1.0:1783520936.263430:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2259 count 1 00000100:00100000:3.0:1783520936.263438:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516310000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993292928:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.263446:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.263446:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026bd4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8d3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.263462:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001a5ca13f/0xd5401cbabc022740 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d50 expref: 52 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.263469:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d3:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.263473:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.263477:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026bd4 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.263483:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993294848 00000100:00100000:0.0:1783520936.263497:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502c9b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993294976:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.263499:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993294848 00010000:00010000:1.0:1783520936.263515:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001a5ca13f/0xd5401cbabc022740 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d50 expref: 52 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.263528:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001a5ca13f/0xd5401cbabc022740 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d50 expref: 52 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.263531:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45027 00000100:00100000:0.0:1783520936.263537:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.263543:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351664a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+953:3626:x1870152993294848:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.263550:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350793ad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993295104:192.168.204.33@tcp:104: 00002000:00080000:2.0:1783520936.263556:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2260 count 1 00010000:00010000:2.0:1783520936.263571:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026bdb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8d4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.263578:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993294976 00000100:00100000:0.0:1783520936.263583:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993294976 00010000:00010000:2.0:1783520936.263584:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a188ee0a/0xd5401cbabc022747 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d5e expref: 52 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.263594:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d4:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.263597:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d4:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.263599:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.263602:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026bdb lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.263616:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45028 00000100:00100000:0.0:1783520936.263625:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502c9a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+954:3627:x1870152993294976:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.263632:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026bb8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.263633:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a188ee0a/0xd5401cbabc022747 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d5e expref: 52 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.263640:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2261 count 1 00010000:00010000:1.0:1783520936.263655:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026bb8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.263657:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026be2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8d5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.263659:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a188ee0a/0xd5401cbabc022747 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d5e expref: 52 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.263678:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026bb8 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.263688:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351664b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993295232:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.263688:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ca09e743/0xd5401cbabc02274e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d6c expref: 52 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.263692:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026bb8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.263711:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d5:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.263715:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.263719:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026be2 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.263720:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1900 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.263725:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.263740:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.263745:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026bb8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.263746:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516649500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993294336:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.263758:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.263763:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026bb8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.263766:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026bbf lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.263779:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026bb8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.263793:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026bb8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.263801:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026bb8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.263815:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350793bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+956:3625:x1870152993293696:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4672us (4749us total) trans 4294976523 rc 0/0 00010000:00010000:0.0:1783520936.263817:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.263822:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.263825:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.263826:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45022 00000004:00080000:2.0:1783520936.263843:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516312a00/2, rc 0, transno 4294976524 00000004:00080000:0.0:1783520936.263854:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351cc7d180/2, rc 0, transno 4294976523 00000100:00100000:2.0:1783520936.263855:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516312a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993292800:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.263863:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351cc7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993293696:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.263869:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435166c8000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993294464:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.263884:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026bc6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.263895:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ae5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+956:3626:x1870152993292800:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5321us (5473us total) trans 4294976524 rc 0/0 00000100:00100000:3.0:1783520936.263904:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45019 00000100:00100000:2.0:1783520936.263920:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502c99c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993295360:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.263940:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1783520936.263947:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.263951:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502c99500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993295488:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.263969:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993295360 00000100:00100000:1.0:1783520936.263974:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993295360 00000100:00100000:0.0:1783520936.263984:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.264007:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993295488 00000100:00100000:2.0:1783520936.264012:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993295488 00000100:00100000:1.0:1783520936.264013:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45029 00000100:00100000:1.0:1783520936.264025:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943516649500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+956:3628:x1870152993295360:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.264038:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2262 count 1 00010000:00010000:0.0:1783520936.264040:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ca09e743/0xd5401cbabc02274e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d6c expref: 52 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.264049:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45030 00010000:00010000:1.0:1783520936.264050:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026be9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8d6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.264055:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ca09e743/0xd5401cbabc02274e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d6c expref: 52 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.264062:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+956:3626:x1870152993295488:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.264070:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000008a89d399/0xd5401cbabc022931 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3021 expref: 52 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.264079:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2263 count 1 00000100:00100000:0.0:1783520936.264084:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a6a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993295616:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.264094:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026bf0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8d7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.264095:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x8d6:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.264102:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.264110:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026be9 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.264122:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80b40 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.264124:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001e14eaed/0xd5401cbabc02275c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d88 expref: 52 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.264133:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d7:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.264137:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.264140:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000008a89d399/0xd5401cbabc022931 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3021 expref: 52 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.264143:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026bf0 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.264163:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000008a89d399/0xd5401cbabc022931 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3021 expref: 52 pid: 35274 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.264177:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001e14eaed/0xd5401cbabc02275c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d88 expref: 52 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.264177:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435166c8000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993295744:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.264194:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001e14eaed/0xd5401cbabc02275c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d88 expref: 52 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.264206:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503849180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993295872:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.264217:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470f00 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.264234:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b16c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.264345:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351664aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993294592:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.264348:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.264379:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026bcd lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.264405:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002489600 00000100:00100000:1.0:1783520936.264409:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002489600 00010000:00010000:1.0:1783520936.264424:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000005fa00a3d/0xd5401cbabc022732 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d34 expref: 53 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.264430:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.264455:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002489472 00000100:00100000:1.0:1783520936.264458:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002489472 00000100:00100000:2.0:1783520936.264463:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36614 00010000:00010000:1.0:1783520936.264468:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008163e037/0xd5401cbabc02272b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d26 expref: 54 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.264476:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a7100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+54:2389:x1870153002489600:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.264484:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:1.0:1783520936.264500:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36615 00010000:00010000:2.0:1783520936.264505:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000005fa00a3d/0xd5401cbabc022732 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d34 expref: 54 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.264511:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a4380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+54:2389:x1870153002489472:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.264524:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.264526:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000005fa00a3d/0xd5401cbabc022732 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8d1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d34 expref: 54 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.264530:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008163e037/0xd5401cbabc02272b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d26 expref: 54 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.264549:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008163e037/0xd5401cbabc02272b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8d0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d26 expref: 54 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.264553:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000005fa00a3d/0xd5401cbabc022732 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8d1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d34 expref: 54 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.264571:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.264575:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008163e037/0xd5401cbabc02272b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8d0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d26 expref: 53 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.264576:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.264584:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026bc6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.264592:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.264594:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026bc6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.264596:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.264602:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026bbf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.264611:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026bbf lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.264622:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.264628:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.264636:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026bc6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.264645:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.264651:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026bc6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.264655:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a4380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+52:2389:x1870153002489472:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 144us (225us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.264656:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a7100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+52:2389:x1870153002489600:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 181us (307us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.264670:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36615 00000100:00100000:2.0:1783520936.264679:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36614 00010000:00010000:0.0:1783520936.264681:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026bc6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.264686:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002489728 00000100:00100000:1.0:1783520936.264690:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002489728 00010000:00010000:0.0:1783520936.264691:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026bc6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.264702:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002c725817/0xd5401cbabc022739 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d42 expref: 51 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.264704:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026bbf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.264710:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.264725:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026bbf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.264739:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.264742:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026bbf lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.264744:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026bc6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.264760:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.264761:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026bbf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.264764:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026bc6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.264779:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026bc6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.264786:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026bc6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.264787:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.264795:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026bc6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.264797:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435166cb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+957:3627:x1870152993294080:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2079us (2280us total) trans 4294976525 rc 0/0 00000100:00100000:0.0:1783520936.264809:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45024 00010000:00010000:1.0:1783520936.264811:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.264815:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026bbf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.264821:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.264825:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026bbf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.264828:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.264841:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026bbf lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.264852:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026bbf lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.264859:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d023bc59/0xd5401cbabc026bbf lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.264878:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351664bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+957:3626:x1870152993293952:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2259us (2406us total) trans 4294976526 rc 0/0 00010000:00010000:1.0:1783520936.264878:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.264882:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.264885:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.264889:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45023 00000004:00080000:0.0:1783520936.264890:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514e9ad80/2, rc 0, transno 4294976526 00000100:00100000:1.0:1783520936.264902:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36616 00000100:00100000:0.0:1783520936.264904:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514e9ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993293952:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.264911:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f1850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+51:2387:x1870153002489728:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.264917:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.264921:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002c725817/0xd5401cbabc022739 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d42 expref: 51 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.264929:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002c725817/0xd5401cbabc022739 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8d2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d42 expref: 51 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.264939:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002c725817/0xd5401cbabc022739 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8d2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d42 expref: 51 pid: 12734 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.264952:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.264955:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d2:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.264957:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993296000:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.264958:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026bcd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.264968:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026bcd lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.264985:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.264991:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.264995:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026bcd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.265008:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026bcd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.265008:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f1850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+50:2387:x1870153002489728:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 99us (369us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.265024:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36616 00010000:00010000:3.0:1783520936.265028:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026bcd lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.265029:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993296000 00000100:00100000:2.0:1783520936.265034:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993296000 00010000:00010000:3.0:1783520936.265036:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026bcd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.265054:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.265066:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45031 00010000:00010000:0.0:1783520936.265074:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:2.0:1783520936.265075:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351664aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+957:3626:x1870152993296000:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.265081:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026bcd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.265083:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435166cbb80/2, rc 0, transno 4294976525 00002000:00080000:2.0:1783520936.265089:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2264 count 1 00010000:00010000:0.0:1783520936.265097:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185000 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520936.265099:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435166cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993294080:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.265104:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026bcd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.265105:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026bf7 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8d8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.265126:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026bcd lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.265130:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006b0db2e9/0xd5401cbabc022763 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d96 expref: 49 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.265146:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d8:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.265149:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026bcd lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.265150:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.265158:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026bf7 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.265164:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d4a09c64/0xd5401cbabc026bcd lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.265173:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351530d500/2, rc 0, transno 4294976527 00000100:00100000:3.0:1783520936.265177:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e3b0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+958:3628:x1870152993294208:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2211us (2280us total) trans 4294976527 rc 0/0 00000100:00100000:1.0:1783520936.265183:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993294208:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.265184:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006b0db2e9/0xd5401cbabc022763 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d96 expref: 49 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.265184:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.265190:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.265191:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1783520936.265194:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45025 00010000:00010000:0.0:1783520936.265195:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.265201:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350626e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993296128:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.265207:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006b0db2e9/0xd5401cbabc022763 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d96 expref: 49 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.265220:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993296256:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.265226:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.265248:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993296128 00000100:00100000:3.0:1783520936.265254:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993296128 00000100:00100000:2.0:1783520936.265273:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1a80 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.265277:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350626d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993296384:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.265291:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45032 00000100:00100000:3.0:1783520936.265306:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350626ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+958:3628:x1870152993296128:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.265306:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00080000:3.0:1783520936.265322:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2265 count 1 00000100:00100000:0.0:1783520936.265327:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993296384 00000100:00100000:0.0:1783520936.265332:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993296384 00010000:00010000:3.0:1783520936.265336:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026bfe lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8d9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.265350:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e5314ee4/0xd5401cbabc02276a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2da4 expref: 49 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.265389:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45033 00000100:00100000:0.0:1783520936.265397:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350626dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+959:3628:x1870152993296384:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:0.0:1783520936.265408:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2266 count 1 00010000:00010000:0.0:1783520936.265420:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c05 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8da:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.265432:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007d9ef3e6/0xd5401cbabc022771 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2db2 expref: 49 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.265440:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8da:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.265443:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8da:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.265446:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c05 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.265460:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d9:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.265466:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.265467:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007d9ef3e6/0xd5401cbabc022771 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2db2 expref: 49 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.265474:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026bfe lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.265482:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007d9ef3e6/0xd5401cbabc022771 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2db2 expref: 49 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.265510:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a5f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993296512:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.265516:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e5314ee4/0xd5401cbabc02276a lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2da4 expref: 49 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.265528:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e5314ee4/0xd5401cbabc02276a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2da4 expref: 49 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.265543:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e3b1880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993296640:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.265543:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350793ad80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993295104:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.265565:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026bd4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.265567:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c803c0 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.265582:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e780 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.265598:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a6a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993295616:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.265613:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026be2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.266010:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351664b480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993295232:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.266027:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026bdb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.266032:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435166c8000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993295744:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.266039:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503849180 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993295872:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.266050:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026be9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.266058:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026bf0 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.266701:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.266727:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.266729:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002489856 00000100:00100000:3.0:1783520936.266735:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002489856 00010000:00010000:3.0:1783520936.266759:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ca09e743/0xd5401cbabc02274e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d6c expref: 50 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.266760:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002490368 00000100:00100000:1.0:1783520936.266760:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.266767:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002490368 00010000:00010000:2.0:1783520936.266789:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001e14eaed/0xd5401cbabc02275c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d88 expref: 51 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.266791:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002490112 00000100:00100000:3.0:1783520936.266797:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002490112 00000100:00100000:1.0:1783520936.266801:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.266806:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36617 00000100:00100000:0.0:1783520936.266815:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435166c8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+52:2388:x1870153002489856:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.266821:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a188ee0a/0xd5401cbabc022747 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d5e expref: 52 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.266824:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.266832:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ca09e743/0xd5401cbabc02274e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d6c expref: 52 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.266841:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ca09e743/0xd5401cbabc02274e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8d5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d6c expref: 52 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.266842:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002489984 00000100:00100000:1.0:1783520936.266843:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36618 00000100:00100000:2.0:1783520936.266848:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002489984 00000100:00100000:3.0:1783520936.266858:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36619 00000100:00100000:1.0:1783520936.266859:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515476a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+53:2390:x1870153002490368:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.266863:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ca09e743/0xd5401cbabc02274e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8d5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d6c expref: 53 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.266867:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001a5ca13f/0xd5401cbabc022740 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d50 expref: 53 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.266870:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.266872:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515474380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+53:2388:x1870153002490112:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.266879:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.266880:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001e14eaed/0xd5401cbabc02275c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d88 expref: 52 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.266886:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.266887:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.266896:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a188ee0a/0xd5401cbabc022747 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d5e expref: 52 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.266897:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026be2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.266902:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001e14eaed/0xd5401cbabc02275c lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8d7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d88 expref: 52 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.266910:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36620 00010000:00010000:3.0:1783520936.266922:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a188ee0a/0xd5401cbabc022747 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8d4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d5e expref: 52 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.266927:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e0f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+52:2388:x1870153002489984:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.266932:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001e14eaed/0xd5401cbabc02275c lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8d7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d88 expref: 52 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.266942:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026be2 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.266945:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.266955:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001a5ca13f/0xd5401cbabc022740 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d50 expref: 52 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.266956:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a188ee0a/0xd5401cbabc022747 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8d4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d5e expref: 51 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.266958:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.266965:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.266967:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.266975:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026bf0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.266986:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.266986:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001a5ca13f/0xd5401cbabc022740 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8d3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d50 expref: 50 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.266989:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026bf0 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.266996:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d4:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.266998:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435166c8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+50:2388:x1870153002489856:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 177us (290us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.267008:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026bdb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.267018:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36617 00010000:00010000:2.0:1783520936.267019:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001a5ca13f/0xd5401cbabc022740 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8d3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d50 expref: 50 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.267036:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026bdb lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.267041:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.267043:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.267049:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.267050:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026be2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.267057:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026bd4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.267061:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026be2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.267062:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515476a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+48:2390:x1870153002490368:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 208us (334us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.267077:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36618 00010000:00010000:2.0:1783520936.267082:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026bd4 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.267086:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026be2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.267101:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026be2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.267109:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.267116:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.267122:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.267128:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:2.0:1783520936.267132:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e0f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+47:2388:x1870153002489984:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 209us (334us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.267135:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026be2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.267143:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.267144:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36620 00000100:00100000:3.0:1783520936.267151:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515474380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+47:2388:x1870153002490112:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 282us (391us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.267155:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb400 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:2.0:1783520936.267161:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002490240 00010000:00010000:0.0:1783520936.267162:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026be2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.267165:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002490240 00010000:00010000:0.0:1783520936.267175:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026be2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.267178:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000008a89d399/0xd5401cbabc022931 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3021 expref: 47 pid: 35274 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.267193:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026be2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.267200:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026be2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.267206:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36619 00010000:00010000:0.0:1783520936.267218:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.267222:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.267226:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.267232:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36621 00000100:00100000:2.0:1783520936.267241:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509174a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+46:2388:x1870153002490240:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.267248:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.267252:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000008a89d399/0xd5401cbabc022931 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8d6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3021 expref: 46 pid: 35274 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.267254:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026bf0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.267267:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026bf0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.267268:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000008a89d399/0xd5401cbabc022931 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8d6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3021 expref: 46 pid: 35274 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.267283:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026bf0 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.267289:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026bf0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.267291:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000008a89d399/0xd5401cbabc022931 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8d6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 12582912->12587007) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3021 expref: 46 pid: 35274 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.267303:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.267309:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.267311:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.267315:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:3.0:1783520936.267319:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520936.267319:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502c9a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+960:3627:x1870152993294976:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3693us (3784us total) trans 4294976528 rc 0/0 00010000:00010000:2.0:1783520936.267320:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026be9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.267333:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.267335:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45028 00010000:00010000:2.0:1783520936.267340:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026be9 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.267386:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.267407:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.267418:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026bf0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.267425:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002490496 00000004:00080000:0.0:1783520936.267425:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502c9b800/2, rc 0, transno 4294976528 00000100:00100000:1.0:1783520936.267430:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002490496 00000100:00100000:2.0:1783520936.267435:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509174a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+46:2388:x1870153002490240:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 195us (296us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.267440:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:0.0:1783520936.267440:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502c9b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993294976:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.267446:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36621 00010000:00010000:3.0:1783520936.267448:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026bf0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.267450:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006b0db2e9/0xd5401cbabc022763 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d96 expref: 46 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.267459:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002490752 00000100:00100000:2.0:1783520936.267464:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002490752 00010000:00010000:3.0:1783520936.267468:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026bf0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.267478:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e5314ee4/0xd5401cbabc02276a lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2da4 expref: 46 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.267482:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026bf0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.267496:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002490624 00000100:00100000:1.0:1783520936.267502:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002490624 00010000:00010000:3.0:1783520936.267506:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026bf0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.267511:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36622 00000100:00100000:0.0:1783520936.267512:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:1.0:1783520936.267521:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007d9ef3e6/0xd5401cbabc022771 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2db2 expref: 47 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.267527:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509175880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+47:2387:x1870153002490496:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.267527:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993296768:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.267529:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.267533:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.267538:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.267542:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.267549:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006b0db2e9/0xd5401cbabc022763 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8d8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d96 expref: 47 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.267556:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36623 00000100:00100000:3.0:1783520936.267569:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36624 00010000:00010000:2.0:1783520936.267570:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006b0db2e9/0xd5401cbabc022763 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d96 expref: 47 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.267571:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.267572:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9436298f2c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+47:2390:x1870153002490752:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520936.267583:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517209500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+47:2389:x1870153002490624:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.267587:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.267597:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e5314ee4/0xd5401cbabc02276a lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8d9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2da4 expref: 47 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.267599:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.267601:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.267609:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007d9ef3e6/0xd5401cbabc022771 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8da:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2db2 expref: 47 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.267610:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.267618:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e5314ee4/0xd5401cbabc02276a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8d9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2da4 expref: 47 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.267621:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026bf7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.267633:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007d9ef3e6/0xd5401cbabc022771 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8da:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2db2 expref: 47 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.267646:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026bf7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8d8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.267650:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.267659:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8d9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.267669:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026bfe lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.267675:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8da:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.267675:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026bfe lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8d9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.267679:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.267686:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8da:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.267691:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c05 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8da:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.267695:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.267697:0:20415:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007d9ef3e6/0xd5401cbabc022771 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8da:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2db2 expref: 47 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.267706:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c05 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8da:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.267710:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509175880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+47:2387:x1870153002490496:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 186us (409us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.267716:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9436298f2c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+46:2390:x1870153002490752:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 150us (385us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.267718:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36622 00000100:00100000:0.0:1783520936.267719:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a5f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993296512:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.267729:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36623 00010000:00010000:2.0:1783520936.267746:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026bdb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.267764:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026bdb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.267768:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c05 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.267771:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026bdb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.267773:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026be9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.267774:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.267780:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c05 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.267795:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026be9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.267797:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026bdb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.267797:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c05 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.267803:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517209500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+44:2389:x1870153002490624:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 221us (443us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.267824:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36624 00010000:00010000:1.0:1783520936.267824:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026be9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.267825:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c05 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.267836:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.267838:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c05 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.267848:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026be9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.267849:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993296768 00000100:00100000:3.0:1783520936.267855:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993296768 00010000:00010000:0.0:1783520936.267856:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1783520936.267868:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1783520936.267869:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.267875:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c05 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.267877:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026bdb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.267887:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45034 00010000:00010000:0.0:1783520936.267889:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.267896:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.267896:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c05 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.267901:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+961:3626:x1870152993296768:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.267903:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026bdb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.267916:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351664a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+961:3626:x1870152993294848:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4375us (4472us total) trans 4294976529 rc 0/0 00010000:00010000:0.0:1783520936.267918:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c05 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.267920:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2267 count 1 00010000:00010000:1.0:1783520936.267920:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026bdb lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.267930:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45027 00010000:00010000:1.0:1783520936.267944:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026bdb lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.267950:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026bdb lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.267953:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c05 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.267960:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026bd4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.267960:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c05 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.267962:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.267964:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.267967:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.267981:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026bd4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.267981:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.267986:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.267991:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026c0c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8db:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.267992:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026be9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.268017:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026bd4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.268017:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.268021:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000006d218c60/0xd5401cbabc022778 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2dc0 expref: 43 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.268024:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026be9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.268037:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026bd4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.268045:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8db:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.268052:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026be9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.268054:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8db:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.268057:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943516649500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+961:3628:x1870152993295360:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4033us (4109us total) trans 4294976530 rc 0/0 00010000:00010000:3.0:1783520936.268066:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026c0c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.268070:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.268073:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45029 00010000:00010000:0.0:1783520936.268080:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026be9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.268088:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007537cc83/0xd5401cbabc026be9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.268089:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.268096:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026bd4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.268100:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502c99c00/2, rc 0, transno 4294976530 00010000:00010000:3.0:1783520936.268102:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.268107:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026bd4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.268111:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502c99c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993295360:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.268111:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.268117:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026bd4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.268118:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.268124:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.268130:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026bd4 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.268136:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026bd4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.268136:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.268145:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ad2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+961:3625:x1870152993294720:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4734us (4836us total) trans 4294976531 rc 0/0 00010000:00010000:3.0:1783520936.268153:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.268154:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45026 00000100:00100000:0.0:1783520936.268154:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351720b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993296896:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.268156:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.268161:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520936.268188:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351530dc00/2, rc 0, transno 4294976531 00010000:00010000:2.0:1783520936.268193:0:35274:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006b0db2e9/0xd5401cbabc022763 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2d96 expref: 43 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.268200:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351530dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993294720:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.268206:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.268207:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993296256:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.268227:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026bf7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.268235:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993296896 00010000:00010000:2.0:1783520936.268238:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026bf7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.268239:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993296896 00000004:00080000:0.0:1783520936.268249:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507ad3480/2, rc 0, transno 4294976529 00010000:00010000:2.0:1783520936.268250:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026bf7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.268262:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026bf7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.268268:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507ad3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993294848:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.268269:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:3.0:1783520936.268276:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000006d218c60/0xd5401cbabc022778 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2dc0 expref: 42 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.268277:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026bf7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.268281:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503849180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993297024:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.268307:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.268312:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520936.268317:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:1.0:1783520936.268319:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026bf7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.268327:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993297280:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.268340:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.268344:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026bf7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.268352:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026bf7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.268354:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000006d218c60/0xd5401cbabc022778 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2dc0 expref: 42 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.268386:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026bf7 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.268391:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.268393:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993297152:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.268402:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000000a37bb7/0xd5401cbabc026bf7 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.268410:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351664aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+962:3626:x1870152993296000:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3336us (3419us total) trans 4294976532 rc 0/0 00000100:00100000:2.0:1783520936.268417:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45031 00010000:00010000:1.0:1783520936.268425:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.268428:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.268432:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.268435:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993297280 00000100:00100000:2.0:1783520936.268439:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993297280 00000100:00100000:3.0:1783520936.268450:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.268457:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00080000:0.0:1783520936.268466:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435121a4700/2, rc 0, transno 4294976532 00000100:00100000:2.0:1783520936.268467:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993297024 00000100:00100000:2.0:1783520936.268472:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993297024 00010000:00010000:3.0:1783520936.268478:0:8408:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e5314ee4/0xd5401cbabc02276a lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2da4 expref: 42 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.268481:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993296000:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.268503:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e3b1880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993296640:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.268508:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45035 00000100:00100000:1.0:1783520936.268516:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45036 00010000:00010000:3.0:1783520936.268519:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026bfe lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.268520:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+963:3626:x1870152993296896:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.268530:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+963:3627:x1870152993297280:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.268535:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026bfe lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.268541:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2268 count 1 00010000:00010000:3.0:1783520936.268543:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026bfe lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.268547:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2270 count 1 00000100:00100000:0.0:1783520936.268550:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:3.0:1783520936.268552:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026bfe lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.268557:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026c13 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8dc:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.268560:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026bfe lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.268562:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993297408:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.268563:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026c1a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8de:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.268588:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002464141d/0xd5401cbabc02277f lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2dce expref: 41 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.268591:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000e0669366/0xd5401cbabc022b8b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f327b expref: 41 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.268600:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.268609:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8dc:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.268610:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x8de:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.268616:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8de:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.268618:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8dc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.268621:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026c1a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.268625:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026c13 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.268641:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000e0669366/0xd5401cbabc022b8b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f327b expref: 41 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.268652:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002464141d/0xd5401cbabc02277f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2dce expref: 41 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.268652:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000e0669366/0xd5401cbabc022b8b lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f327b expref: 41 pid: 8410 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.268668:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+964:3626:x1870152993295488:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4608us (4683us total) trans 4294976533 rc 0/0 00000100:00100000:1.0:1783520936.268677:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435166c8000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993297536:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.268682:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45030 00010000:00010000:2.0:1783520936.268683:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002464141d/0xd5401cbabc02277f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2dce expref: 41 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.268697:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993297664:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.268699:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993297408 00000004:00080000:0.0:1783520936.268703:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502c99500/2, rc 0, transno 4294976533 00000100:00100000:3.0:1783520936.268704:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993297408 00000100:00100000:0.0:1783520936.268717:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502c99500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993295488:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.268723:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470a80 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.268727:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45037 00000100:00100000:2.0:1783520936.268737:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.268748:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45038 00000100:00100000:1.0:1783520936.268757:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+964:3626:x1870152993297408:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.268769:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517208e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+964:3628:x1870152993297024:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.268771:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2271 count 1 00000100:00100000:0.0:1783520936.268774:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993297792:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.268785:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026c21 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8df:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.268795:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000c3f1914d/0xd5401cbabc022794 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2df8 expref: 41 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.268796:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.268810:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.268812:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8df:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.268816:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8df:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.268821:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026c21 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.268833:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.268840:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026bfe lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.268846:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000c3f1914d/0xd5401cbabc022794 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2df8 expref: 41 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.268847:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993297792 00000100:00100000:2.0:1783520936.268855:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993297792 00010000:00010000:0.0:1783520936.268856:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.268862:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026bfe lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.268865:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000c3f1914d/0xd5401cbabc022794 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2df8 expref: 41 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.268876:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350626ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+966:3628:x1870152993296128:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3575us (3649us total) trans 4294976534 rc 0/0 00000100:00100000:2.0:1783520936.268889:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45039 00000100:00100000:1.0:1783520936.268891:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e0f5c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993297920:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.268899:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45032 00000100:00100000:2.0:1783520936.268902:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+966:3626:x1870152993297792:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.268904:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026bfe lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.268918:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2272 count 1 00000004:00080000:1.0:1783520936.268919:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350626e680/2, rc 0, transno 4294976534 00010000:00010000:0.0:1783520936.268920:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026bfe lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.268926:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2269 count 1 00000100:00100000:1.0:1783520936.268932:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350626e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993296128:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.268934:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026c28 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8e0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.268936:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000042ef1cd5/0xd5401cbabc026bfe lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.268939:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c2f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8dd:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.268961:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000001d578fa8/0xd5401cbabc02279b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e06 expref: 41 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.268965:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:00010000:0.0:1783520936.268968:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.268972:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.268973:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e0:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.268976:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.268977:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e0:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.268978:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515476a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993298048:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.268986:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026c28 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.269007:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d9434dd7/0xd5401cbabc022786 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ddc expref: 41 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.269014:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8dd:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.269017:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8dd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.269022:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c2f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.269023:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000001d578fa8/0xd5401cbabc02279b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e06 expref: 41 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.269032:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.269051:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000001d578fa8/0xd5401cbabc02279b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e06 expref: 41 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.269053:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d9434dd7/0xd5401cbabc022786 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ddc expref: 41 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.269073:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993298048 00000100:00100000:0.0:1783520936.269080:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993298048 00000100:00100000:2.0:1783520936.269084:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11f800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993298176:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.269085:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d9434dd7/0xd5401cbabc022786 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ddc expref: 41 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.269088:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.269112:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45040 00000100:00100000:0.0:1783520936.269122:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943515474380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+966:3628:x1870152993298048:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.269123:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1780 going to sleep for 11 seconds 00002000:00080000:0.0:1783520936.269136:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2273 count 1 00010000:00010000:0.0:1783520936.269149:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026c36 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8e1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.269161:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ff53a785/0xd5401cbabc0227a2 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e14 expref: 41 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.269168:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e3b1880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993298304:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.269173:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e1:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.269177:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.269183:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026c36 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.269204:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ff53a785/0xd5401cbabc0227a2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e14 expref: 41 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.269205:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e840 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.269219:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ff53a785/0xd5401cbabc0227a2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e14 expref: 41 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.269231:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575ca80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993298432:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.269264:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c803c0 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.269355:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350626dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+967:3628:x1870152993296384:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3958us (4055us total) trans 4294976535 rc 0/0 00000100:00100000:0.0:1783520936.269389:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45033 00000004:00080000:1.0:1783520936.269430:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350626d180/2, rc 0, transno 4294976535 00000100:00100000:1.0:1783520936.269447:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350626d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993296384:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.269602:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b0700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993297152:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.269615:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026c0c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.269936:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435166c8000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993297536:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.269951:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026c1a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.270025:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993297664:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.270043:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026c13 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.270100:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e0f5c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993297920:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.270116:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026c21 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.270127:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993298560:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.270158:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.270251:0:8408:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993298560 00000100:00100000:3.0:1783520936.270256:0:8408:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993298560 00000100:00100000:3.0:1783520936.270289:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45041 00000100:00100000:3.0:1783520936.270298:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943635739450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+967:3628:x1870152993298560:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520936.270312:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2274 count 1 00010000:00010000:3.0:1783520936.270326:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026c3d lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8e2:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.270338:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000346ad523/0xd5401cbabc0227a9 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e22 expref: 41 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.270351:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e2:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.270355:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e2:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.270379:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e3b1880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993298304:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.270383:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026c3d lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.270414:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c2f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.270439:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000346ad523/0xd5401cbabc0227a9 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e22 expref: 41 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.270452:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000346ad523/0xd5401cbabc0227a9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e22 expref: 41 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.270464:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e0f5c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993298688:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.270537:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.270556:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.270599:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002491392 00000100:00100000:1.0:1783520936.270605:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002491392 00010000:00010000:1.0:1783520936.270629:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000001d578fa8/0xd5401cbabc02279b lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e06 expref: 42 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.270661:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36625 00000100:00100000:0.0:1783520936.270680:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11f800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993298176:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.270689:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+42:2389:x1870153002491392:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.270703:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.270711:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.270719:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000001d578fa8/0xd5401cbabc02279b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8e0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e06 expref: 42 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.270727:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026c28 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8e0:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.270755:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000001d578fa8/0xd5401cbabc02279b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8e0:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e06 expref: 42 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.270768:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.270787:0:6478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001d578fa8/0xd5401cbabc02279b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8e0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e06 expref: 42 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.270788:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002490880 00000100:00100000:3.0:1783520936.270794:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002490880 00010000:00010000:1.0:1783520936.270808:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.270814:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.270818:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000006d218c60/0xd5401cbabc022778 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2dc0 expref: 42 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.270820:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575ca80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993298432:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.270833:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026c28 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.270837:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.270845:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026c36 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.270871:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002491136 00010000:00010000:1.0:1783520936.270874:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026c28 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.270877:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.270880:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002491136 00010000:00010000:3.0:1783520936.270907:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002464141d/0xd5401cbabc02277f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2dce expref: 43 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.270911:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002491520 00000100:00100000:1.0:1783520936.270917:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002491520 00000100:00100000:2.0:1783520936.270924:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.270930:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002491008 00000100:00100000:3.0:1783520936.270934:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002491008 00010000:00010000:1.0:1783520936.270941:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d9434dd7/0xd5401cbabc022786 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ddc expref: 45 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.270953:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000e0669366/0xd5401cbabc022b8b lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f327b expref: 45 pid: 8410 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.270954:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.270967:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002491264 00000100:00100000:1.0:1783520936.270972:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002491264 00000100:00100000:3.0:1783520936.270975:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002491648 00000100:00100000:3.0:1783520936.270979:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002491648 00000100:00100000:2.0:1783520936.270985:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36626 00010000:00010000:1.0:1783520936.270992:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000c3f1914d/0xd5401cbabc022794 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2df8 expref: 47 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.270997:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ff53a785/0xd5401cbabc0227a2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e14 expref: 47 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.271013:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351664b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+47:2387:x1870153002490880:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.271032:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36627 00010000:00010000:2.0:1783520936.271039:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:3.0:1783520936.271040:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36628 00000100:00100000:1.0:1783520936.271045:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350793ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+47:2387:x1870153002491136:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.271051:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000006d218c60/0xd5401cbabc022778 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8db:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2dc0 expref: 47 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.271055:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505875c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+47:2387:x1870153002491520:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.271057:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.271067:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002464141d/0xd5401cbabc02277f lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8dc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2dce expref: 47 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.271072:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.271078:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000006d218c60/0xd5401cbabc022778 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8db:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2dc0 expref: 47 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.271083:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d9434dd7/0xd5401cbabc022786 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8dd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ddc expref: 47 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.271085:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002464141d/0xd5401cbabc02277f lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8dc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2dce expref: 47 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.271115:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006d218c60/0xd5401cbabc022778 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8db:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2dc0 expref: 47 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.271116:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d9434dd7/0xd5401cbabc022786 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8dd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ddc expref: 47 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.271120:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002464141d/0xd5401cbabc02277f lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8dc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2dce expref: 47 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.271153:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8db:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.271153:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8dc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.271154:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d9434dd7/0xd5401cbabc022786 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8dd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ddc expref: 45 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.271162:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8db:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.271171:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026c0c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.271173:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8dd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.271173:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8dc:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.271183:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8dd:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.271185:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026c13 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.271187:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026c0c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8db:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.271191:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c2f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.271214:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026c13 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.271216:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c2f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.271235:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.271239:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.271286:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351664b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+44:2387:x1870153002490880:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 274us (590us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.271286:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350793ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+44:2387:x1870153002491136:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 240us (517us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.271286:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026c0c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.271328:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36626 00010000:00010000:0.0:1783520936.271338:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026c0c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.271341:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36627 00010000:00010000:3.0:1783520936.271349:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.271411:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026c0c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.271420:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36629 00010000:00010000:0.0:1783520936.271424:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026c0c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.271437:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505875c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+42:2387:x1870153002491520:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 376us (606us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.271442:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36630 00000100:00100000:2.0:1783520936.271447:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505874e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+42:2387:x1870153002491008:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1783520936.271462:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357f050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+42:2387:x1870153002491264:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1783520936.271473:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36628 00010000:00010000:2.0:1783520936.271474:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.271500:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000e0669366/0xd5401cbabc022b8b lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8de:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f327b expref: 42 pid: 8410 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.271503:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.271512:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000c3f1914d/0xd5401cbabc022794 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8df:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2df8 expref: 41 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.271514:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36631 00010000:00010000:0.0:1783520936.271515:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.271526:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000e0669366/0xd5401cbabc022b8b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8de:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f327b expref: 41 pid: 8410 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.271538:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505875f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+41:2389:x1870153002491648:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.271542:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000c3f1914d/0xd5401cbabc022794 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8df:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2df8 expref: 41 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.271548:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.271563:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026c0c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.271567:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.271583:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ff53a785/0xd5401cbabc0227a2 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8e1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e14 expref: 41 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.271596:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e0669366/0xd5401cbabc022b8b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8de:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 58720256->58724351) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f327b expref: 41 pid: 8410 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.271610:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c3f1914d/0xd5401cbabc022794 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8df:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2df8 expref: 41 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.271618:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.271635:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026c0c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.271639:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ff53a785/0xd5401cbabc0227a2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8e1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e14 expref: 41 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.271645:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8de:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.271646:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8df:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.271663:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ff53a785/0xd5401cbabc0227a2 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8e1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e14 expref: 39 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.271666:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8de:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.271668:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8df:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.271691:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026c1a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.271693:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026c21 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.271708:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026c0c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.271717:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.271738:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.271741:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026c1a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.271750:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026c21 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.271765:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026c36 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.271780:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026c0c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.271799:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026c36 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.271804:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.271805:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.271806:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a4f07cfe/0xd5401cbabc026c0c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.271832:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357f050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+38:2387:x1870153002491264:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 374us (906us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.271840:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.271847:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505874e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+38:2387:x1870153002491008:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 406us (971us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.271847:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36630 00010000:00010000:0.0:1783520936.271853:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.271859:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.271867:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36629 00010000:00010000:0.0:1783520936.271871:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1783520936.271879:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505875f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+37:2389:x1870153002491648:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 335us (918us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.271888:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36631 00010000:00010000:2.0:1783520936.271901:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c2f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.271912:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c2f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.271922:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c2f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.271930:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c2f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.271955:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.271963:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.271971:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c2f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.271996:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.272038:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.272044:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c2f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.272054:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c2f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.272070:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c2f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.272080:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c2f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.272108:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002491776 00010000:00010000:3.0:1783520936.272114:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.272116:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002491776 00010000:00010000:3.0:1783520936.272118:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.272127:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.272145:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000346ad523/0xd5401cbabc0227a9 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e22 expref: 36 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.272145:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+969:3626:x1870152993296768:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4240us (4571us total) trans 4294976536 rc 0/0 00000100:00100000:2.0:1783520936.272154:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517208e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+969:3628:x1870152993297024:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3413us (3692us total) trans 4294976537 rc 0/0 00000100:00100000:0.0:1783520936.272186:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45034 00010000:00010000:3.0:1783520936.272189:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026c1a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.272203:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026c1a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.272203:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45037 00010000:00010000:3.0:1783520936.272217:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026c1a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.272217:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36632 00010000:00010000:3.0:1783520936.272236:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026c1a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.272237:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509174a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+36:2388:x1870153002491776:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.272256:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.272264:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.272264:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000346ad523/0xd5401cbabc0227a9 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8e2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e22 expref: 36 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.272270:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435121a6a00/2, rc 0, transno 4294976536 00010000:00010000:2.0:1783520936.272278:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.272291:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026c1a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.272295:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000346ad523/0xd5401cbabc0227a9 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8e2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e22 expref: 36 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.272300:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993296768:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.272325:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.272330:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026c1a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.272339:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026c1a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.272339:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.272349:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.272382:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026c3d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.272386:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026c1a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.272402:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026c3d lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8e2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.272403:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f5b84dd8/0xd5401cbabc026c1a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.272405:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+968:3627:x1870152993297280:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3879us (4015us total) trans 4294976538 rc 0/0 00000100:00100000:3.0:1783520936.272426:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45036 00010000:00010000:2.0:1783520936.272446:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.272446:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.272451:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.272456:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.272477:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1783520936.272501:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509174a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+36:2388:x1870153002491776:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 256us (490us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.272511:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351575ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993298816:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.272523:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36632 00000004:00080000:3.0:1783520936.272532:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435121a7100/2, rc 0, transno 4294976538 00000100:00100000:3.0:1783520936.272549:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993297280:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.272562:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026c21 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.272576:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.272593:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026c21 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.272605:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026c21 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.272615:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026c21 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.272618:0:35276:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993298816 00000100:00100000:2.0:1783520936.272624:0:35276:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993298816 00010000:00010000:1.0:1783520936.272647:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000004:00080000:0.0:1783520936.272658:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503849180/2, rc 0, transno 4294976537 00000100:00100000:3.0:1783520936.272665:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1783520936.272665:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45042 00000100:00100000:2.0:1783520936.272679:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351575e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+968:3626:x1870152993298816:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.272683:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e3b1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993298944:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.272684:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503849180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993297024:0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.272697:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2275 count 1 00010000:00010000:2.0:1783520936.272717:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026c44 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8e3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.272721:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.272731:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000efe3b6e0/0xd5401cbabc0227b0 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e30 expref: 35 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.272731:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026c21 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.272753:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.272757:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+969:3626:x1870152993297408:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3995us (4154us total) trans 4294976539 rc 0/0 00010000:00010000:0.0:1783520936.272789:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.272791:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e3:0x0].0x0 since=38s v=(0 0) 00000100:00100000:1.0:1783520936.272796:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45038 00010000:00010000:2.0:1783520936.272801:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.272801:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026c21 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.272816:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993298944 00010000:00010000:2.0:1783520936.272820:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026c44 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.272828:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993298944 00010000:00010000:0.0:1783520936.272845:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026c21 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.272864:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026c21 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.272875:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000efe3b6e0/0xd5401cbabc0227b0 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e30 expref: 35 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.272876:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45043 00000100:00100000:3.0:1783520936.272898:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993299072:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.272899:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435166c8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+969:3625:x1870152993298944:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.272904:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000073e2b95/0xd5401cbabc026c21 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.272925:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2276 count 1 00000100:00100000:3.0:1783520936.272939:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.272939:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000efe3b6e0/0xd5401cbabc0227b0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e30 expref: 35 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.272953:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026c4b lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8e4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.272963:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943505877800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993299200:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.272969:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.272975:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009cb7e7ab/0xd5401cbabc0227b7 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e3e expref: 35 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.272978:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.272988:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.272989:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e4:0x0].0x0 since=38s v=(0 0) 00000100:00100000:3.0:1783520936.272992:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993299072 00010000:00010000:1.0:1783520936.272996:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e4:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.272998:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993299072 00010000:00010000:1.0:1783520936.273001:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026c4b lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.273031:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009cb7e7ab/0xd5401cbabc0227b7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e3e expref: 35 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.273032:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b4700/2, rc 0, transno 4294976539 00000100:00100000:2.0:1783520936.273040:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1000 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.273042:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45044 00010000:00010000:1.0:1783520936.273056:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009cb7e7ab/0xd5401cbabc0227b7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e3e expref: 35 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.273059:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435032e8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+970:3627:x1870152993299072:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.273064:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993297408:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.273094:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509175880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993299328:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.273126:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026c36 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.273140:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026c36 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.273143:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470a80 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.273151:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026c36 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.273154:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2277 count 1 00000100:00100000:2.0:1783520936.273160:0:3627:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00010000:00010000:0.0:1783520936.273174:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026c36 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.273178:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026c52 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8e5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.273178:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435032eb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993299456:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.273180:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.273208:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000023d9d77d/0xd5401cbabc0227be lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e4c expref: 35 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.273213:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+35:2389:x1870153002491392:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 2540us (2654us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.273232:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.273236:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36625 00010000:00010000:3.0:1783520936.273240:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e5:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.273246:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.273251:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026c52 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.273255:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993299456 00000100:00100000:0.0:1783520936.273265:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993299456 00010000:00010000:3.0:1783520936.273286:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000023d9d77d/0xd5401cbabc0227be lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e4c expref: 34 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.273300:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000023d9d77d/0xd5401cbabc0227be lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e4c expref: 34 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.273315:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350d166a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993299584:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.273327:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45045 00000100:00100000:0.0:1783520936.273342:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503a6bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+971:3627:x1870152993299456:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.273355:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00002000:00080000:0.0:1783520936.273432:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2278 count 1 00010000:00010000:0.0:1783520936.273451:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c59 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8e6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.273464:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000043c75256/0xd5401cbabc022a7a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f316a expref: 34 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.273470:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x8e6:0x0].0x0 since=37s v=(0 0) 00010000:00010000:0.0:1783520936.273475:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.273480:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c59 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.273500:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026c13 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.273513:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026c13 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.273524:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000043c75256/0xd5401cbabc022a7a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f316a expref: 34 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.273526:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026c13 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.273562:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026c13 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.273563:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000043c75256/0xd5401cbabc022a7a lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f316a expref: 34 pid: 8408 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.273608:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993299712:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.273623:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.273642:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.273650:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026c13 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.273661:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.273668:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026c13 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.273678:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026c13 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.273681:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c803c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.273705:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026c13 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.273716:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026c13 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.273726:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.273734:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.273740:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.273742:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026c36 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.273748:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.273754:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+972:3626:x1870152993296896:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5233us (5548us total) trans 4294976540 rc 0/0 00010000:00010000:2.0:1783520936.273756:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.273759:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeac00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520936.273769:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45035 00000004:00080000:0.0:1783520936.273773:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351720b480/2, rc 0, transno 4294976540 00000100:00100000:0.0:1783520936.273794:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351720b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993296896:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.273820:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026c36 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.273832:0:8408:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000346ad523/0xd5401cbabc0227a9 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e22 expref: 34 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.273836:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026c36 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.273863:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e0f5c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993298688:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.273868:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:3.0:1783520936.273873:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026c36 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.273884:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503a6b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993299840:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.273888:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a07e3112/0xd5401cbabc026c36 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.273888:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026c3d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.273918:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026c3d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.273927:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026c3d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.273928:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.273934:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.273939:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.273940:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.273951:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026c3d lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.273961:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026c3d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.273980:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993299840 00010000:00010000:1.0:1783520936.273987:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1783520936.273989:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993299840 00010000:00010000:2.0:1783520936.274003:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.274010:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026c3d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.274020:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5e00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.274025:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026c3d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.274036:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026c3d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.274048:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026c3d lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.274055:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000052af5769/0xd5401cbabc026c3d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.274078:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.274078:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943515474380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+974:3628:x1870152993298048:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4956us (5048us total) trans 4294976541 rc 0/0 00010000:00010000:2.0:1783520936.274086:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.274096:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.274096:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45040 00000100:00100000:1.0:1783520936.274106:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943635739450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+974:3628:x1870152993298560:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3804us (3941us total) trans 4294976542 rc 0/0 00000100:00100000:1.0:1783520936.274117:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45041 00000100:00100000:1.0:1783520936.274149:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45046 00000004:00080000:0.0:1783520936.274158:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e11ce00/2, rc 0, transno 4294976542 00000100:00100000:1.0:1783520936.274163:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e11f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+972:3626:x1870152993299840:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.274176:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993298560:0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.274182:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2279 count 1 00010000:00010000:1.0:1783520936.274203:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026c60 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8e7:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.274206:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943515476a00/2, rc 0, transno 4294976541 00010000:00010000:1.0:1783520936.274227:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002e08b506/0xd5401cbabc0227cc lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e68 expref: 33 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.274230:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515476a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993298048:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.274231:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:1.0:1783520936.274252:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e7:0x0].0x0 since=38s v=(0 0) 00000100:00100000:3.0:1783520936.274253:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503a6a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993299968:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.274261:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e7:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.274266:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503a68a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993300096:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.274268:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026c60 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.274279:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026c28 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.274292:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026c28 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.274300:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002e08b506/0xd5401cbabc0227cc lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e68 expref: 33 pid: 9151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.274307:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.274327:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026c28 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.274338:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993299968 00010000:00010000:0.0:1783520936.274338:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026c28 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.274339:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002e08b506/0xd5401cbabc0227cc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e68 expref: 33 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.274349:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993299968 00000100:00100000:1.0:1783520936.274408:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e0f5c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993300224:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.274420:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45047 00000100:00100000:2.0:1783520936.274431:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350794df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+973:3625:x1870152993299968:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.274448:0:12734:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2280 count 1 00010000:00010000:2.0:1783520936.274462:0:12734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026c67 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8e8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.274476:0:12734:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000009c5b3794/0xd5401cbabc0227d3 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e76 expref: 33 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.274480:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.274484:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f000 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.274506:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e8:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.274511:0:12734:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.274512:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.274525:0:12734:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026c67 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.274525:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026c28 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.274543:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.274571:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.274576:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026c28 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.274589:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026c28 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.274591:0:12734:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000009c5b3794/0xd5401cbabc0227d3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e76 expref: 33 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.274630:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026c28 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.274631:0:12734:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000009c5b3794/0xd5401cbabc0227d3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e76 expref: 33 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.274655:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007aa431df/0xd5401cbabc026c28 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.274667:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.274700:0:12734:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503a69500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993300352:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.274709:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.274715:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.274720:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.274721:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.274746:0:12734:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1780 going to sleep for 11 seconds 00000100:00100000:2.0:1783520936.274777:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002491904 00000100:00100000:2.0:1783520936.274782:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002491904 00000100:00100000:3.0:1783520936.274807:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350d166a00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993299584:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.274809:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000efe3b6e0/0xd5401cbabc0227b0 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e30 expref: 34 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.274828:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:3.0:1783520936.274834:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026c52 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.274844:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002492032 00000100:00100000:2.0:1783520936.274850:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002492032 00000100:00100000:0.0:1783520936.274872:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+974:3626:x1870152993297792:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5969us (6061us total) trans 4294976543 rc 0/0 00000100:00100000:3.0:1783520936.274874:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002492160 00010000:00010000:2.0:1783520936.274875:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000023d9d77d/0xd5401cbabc0227be lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e4c expref: 35 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.274886:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002492160 00000100:00100000:0.0:1783520936.274905:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45039 00000100:00100000:1.0:1783520936.274912:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.274925:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002492288 00000100:00100000:2.0:1783520936.274931:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002492288 00010000:00010000:2.0:1783520936.274952:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000043c75256/0xd5401cbabc022a7a lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x8e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f316a expref: 37 pid: 8408 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.274955:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36633 00010000:00010000:3.0:1783520936.274957:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009cb7e7ab/0xd5401cbabc0227b7 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e3e expref: 37 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.274979:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517209500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+37:2387:x1870153002491904:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.275000:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.275009:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000efe3b6e0/0xd5401cbabc0227b0 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8e3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e30 expref: 37 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.275022:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36634 00010000:00010000:1.0:1783520936.275026:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000efe3b6e0/0xd5401cbabc0227b0 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8e3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e30 expref: 37 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.275037:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36635 00000100:00100000:3.0:1783520936.275043:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351664b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+37:2387:x1870153002492032:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.275050:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e3:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.275053:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350793ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+37:2387:x1870153002492160:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.275054:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.275055:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.275065:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000023d9d77d/0xd5401cbabc0227be lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8e5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e4c expref: 37 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.275067:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026c44 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.275069:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.275079:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009cb7e7ab/0xd5401cbabc0227b7 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8e4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e3e expref: 37 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.275083:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026c44 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8e3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.275095:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009cb7e7ab/0xd5401cbabc0227b7 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8e4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e3e expref: 37 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.275107:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000004:00080000:0.0:1783520936.275108:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b4000/2, rc 0, transno 4294976543 00010000:00010000:2.0:1783520936.275111:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.275118:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.275122:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026c4b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.275128:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026c4b lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8e4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.275133:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993297792:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.275151:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517209500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+37:2387:x1870153002491904:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 174us (489us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.275152:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.275160:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000023d9d77d/0xd5401cbabc0227be lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8e5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e4c expref: 37 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.275164:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36633 00000100:00100000:2.0:1783520936.275179:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350793ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+37:2387:x1870153002492160:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 132us (353us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.275183:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000023d9d77d/0xd5401cbabc0227be lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8e5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e4c expref: 36 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.275189:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36635 00000100:00100000:1.0:1783520936.275201:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36636 00010000:00010000:3.0:1783520936.275203:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.275209:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.275214:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026c52 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.275215:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505875c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+34:2388:x1870153002492288:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.275230:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026c52 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.275232:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:0.0:1783520936.275233:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943507acf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993300480:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.275251:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000043c75256/0xd5401cbabc022a7a lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x8e6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f316a expref: 34 pid: 8408 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.275270:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000043c75256/0xd5401cbabc022a7a lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8e6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f316a expref: 34 pid: 8408 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.275282:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.275287:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e6:0x0].0x0 since=0s v=(1 0) 00010000:00010000:1.0:1783520936.275293:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e6:0x0].0x0 since=0s v=(1 0) 00000100:00100000:0.0:1783520936.275303:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.275304:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c59 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.275311:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351664b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+34:2387:x1870153002492032:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 276us (599us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.275326:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36634 00010000:00010000:1.0:1783520936.275327:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c59 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8e6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.275349:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.275429:0:35276:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000efe3b6e0/0xd5401cbabc0227b0 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e30 expref: 33 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.275435:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505875c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+33:2388:x1870153002492288:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 221us (530us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.275457:0:9151:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000043c75256/0xd5401cbabc022a7a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 37748736->37752831) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f316a expref: 32 pid: 8408 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.275458:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943505877800 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993299200:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.275458:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36636 00010000:00010000:2.0:1783520936.275506:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026c44 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.275507:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993300096 00000100:00100000:0.0:1783520936.275508:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11dc00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993299712:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.275517:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993300096 00010000:00010000:2.0:1783520936.275535:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026c44 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.275540:0:35275:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009cb7e7ab/0xd5401cbabc0227b7 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e3e expref: 30 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.275547:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c59 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.275548:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026c44 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.275574:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509175880 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993299328:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.275598:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c59 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.275601:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026c44 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.275604:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026c4b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.275639:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c59 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.275659:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026c44 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.275692:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c59 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.275702:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026c4b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.275708:0:8409:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993300480 00010000:00010000:2.0:1783520936.275709:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.275714:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c59 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.275717:0:8409:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993300480 00010000:00010000:2.0:1783520936.275724:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.275736:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026c44 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.275748:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026c4b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.275753:0:8409:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45048 00000100:00100000:3.0:1783520936.275770:0:8409:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351313a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+975:3627:x1870152993300096:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.275773:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.275775:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1783520936.275777:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026c4b lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.275786:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026c44 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.275807:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.275809:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026c4b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.275816:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c59 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.275836:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026c44 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.275840:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.275846:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c59 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.275849:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.275860:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026c44 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.275861:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c59 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.275877:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026c44 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.275881:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c59 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.275890:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000476b7781/0xd5401cbabc026c59 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.275894:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.275898:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026c4b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.275906:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.275909:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026c4b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.275911:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.275918:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.275919:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503a6bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+976:3627:x1870152993299456:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2575us (2690us total) trans 4294976544 rc 0/0 00010000:00010000:2.0:1783520936.275938:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026c4b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.275945:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45045 00010000:00010000:2.0:1783520936.275961:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026c4b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.275969:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026c4b lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.276000:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.276004:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520936.276008:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435032eb800/2, rc 0, transno 4294976544 00010000:00010000:2.0:1783520936.276009:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.276009:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435166c8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+976:3625:x1870152993298944:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3115us (3262us total) trans 4294976545 rc 0/0 00010000:00010000:2.0:1783520936.276023:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.276027:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45043 00000100:00100000:0.0:1783520936.276032:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435032eb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993299456:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.276058:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45049 00000100:00100000:2.0:1783520936.276063:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:1.0:1783520936.276072:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517244e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+975:3628:x1870152993300480:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:1.0:1783520936.276088:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2282 count 1 00000100:00100000:0.0:1783520936.276089:0:3626:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1783520936.276103:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e11dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993300608:0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.276104:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026c6e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8ea:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.276121:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002492416 00000100:00100000:2.0:1783520936.276126:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002492416 00010000:00010000:1.0:1783520936.276127:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000002806103e/0xd5401cbabc0227e1 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e92 expref: 29 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.276143:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ea:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.276144:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002e08b506/0xd5401cbabc0227cc lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e68 expref: 30 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.276154:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ea:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.276162:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1783520936.276177:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026c6e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.276203:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36637 00010000:00010000:1.0:1783520936.276209:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000002806103e/0xd5401cbabc0227e1 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e92 expref: 30 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.276226:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357f450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+30:2387:x1870153002492416:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.276226:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000002806103e/0xd5401cbabc0227e1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e92 expref: 30 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.276232:0:178496:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993300608 00000100:00100000:0.0:1783520936.276241:0:178496:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993300608 00010000:00010000:2.0:1783520936.276246:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.276255:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002e08b506/0xd5401cbabc0227cc lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8e7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e68 expref: 30 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.276257:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943505874e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993300736:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.276276:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002e08b506/0xd5401cbabc0227cc lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8e7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e68 expref: 30 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.276285:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45050 00010000:00010000:2.0:1783520936.276294:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e7:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.276299:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e7:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.276305:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435038ed180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+976:3626:x1870152993300608:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.276306:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026c60 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.276316:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.276318:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff9435124703c0 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.276333:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026c60 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8e7:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.276353:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2283 count 1 00010000:00010000:2.0:1783520936.276354:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.276408:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026c75 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8eb:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.276426:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a1d03729/0xd5401cbabc0227e8 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ea0 expref: 30 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.276435:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8eb:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.276439:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8eb:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.276446:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357f450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+30:2387:x1870153002492416:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 232us (381us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.276446:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026c75 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.276476:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36637 00010000:00010000:0.0:1783520936.276518:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a1d03729/0xd5401cbabc0227e8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ea0 expref: 29 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.276524:0:8408:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002e08b506/0xd5401cbabc0227cc lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e68 expref: 29 pid: 9151 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.276537:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a1d03729/0xd5401cbabc0227e8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ea0 expref: 29 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.276570:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc0000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993300864:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.276580:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e0f5c00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993300224:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.276581:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.276610:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026c60 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.276623:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026c60 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.276637:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026c60 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.276641:0:12734:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503a69500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_004:lustre-OST0001_UUID:12734:1870152993300352:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.276642:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80e40 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.276660:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026c60 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.276669:0:12734:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026c67 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.276671:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026c60 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.276703:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.276725:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.276733:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026c60 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.276744:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbbe00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.276749:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026c60 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.276757:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026c60 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.276778:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026c60 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.276786:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000e2348a43/0xd5401cbabc026c60 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.276792:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e3b1880/2, rc 0, transno 4294976545 00010000:00010000:0.0:1783520936.276812:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.276816:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.276822:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e11f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+977:3626:x1870152993299840:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2661us (2888us total) trans 4294976546 rc 0/0 00010000:00010000:0.0:1783520936.276823:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1783520936.276836:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45046 00000100:00100000:3.0:1783520936.276858:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e3b1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993298944:0@lo:6:osp-syn-1-0.0 00000004:00080000:0.0:1783520936.276875:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503a6b800/2, rc 0, transno 4294976546 00000100:00100000:0.0:1783520936.276886:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503a6b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993299840:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.276898:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351575e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+977:3626:x1870152993298816:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4215us (4320us total) trans 4294976547 rc 0/0 00000100:00100000:2.0:1783520936.276907:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45042 00000004:00080000:0.0:1783520936.276908:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351575ca80/2, rc 0, transno 4294976547 00000100:00100000:0.0:1783520936.276921:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351575ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993298816:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.277012:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513139c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993300992:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.277060:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002492544 00000100:00100000:0.0:1783520936.277060:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.277072:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002492544 00010000:00010000:3.0:1783520936.277099:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000009c5b3794/0xd5401cbabc0227d3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e76 expref: 29 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.277113:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993300992 00000100:00100000:2.0:1783520936.277119:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993300992 00000100:00100000:0.0:1783520936.277143:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503372300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993301248:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.277148:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36638 00000100:00100000:2.0:1783520936.277156:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45051 00000100:00100000:2.0:1783520936.277170:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350793ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+977:3626:x1870152993300992:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.277183:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2284 count 1 00000100:00100000:0.0:1783520936.277183:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.277206:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026c7c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8ec:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.277219:0:3628:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1783520936.277223:0:20140:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993301248 00010000:00010000:2.0:1783520936.277227:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000cdcdc012/0xd5401cbabc0227f6 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2eae expref: 29 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.277230:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350d166a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993301120:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.277231:0:20140:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993301248 00000100:00100000:3.0:1783520936.277241:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517333480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+29:2388:x1870153002492544:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.277258:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ec:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.277262:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.277264:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ec:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.277274:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000009c5b3794/0xd5401cbabc0227d3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8e8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e76 expref: 29 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.277274:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026c7c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.277289:0:20140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45052 00000100:00100000:0.0:1783520936.277296:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.277310:0:20140:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350793bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+978:3628:x1870152993301248:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.277323:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000009c5b3794/0xd5401cbabc0227d3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8e8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e76 expref: 29 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.277328:0:20140:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2286 count 1 00010000:00010000:2.0:1783520936.277340:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000cdcdc012/0xd5401cbabc0227f6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2eae expref: 29 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.277344:0:20140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026c83 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8ee:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.277353:0:35282:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993301120 00010000:00010000:1.0:1783520936.277423:0:20140:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000003a53433c/0xd5401cbabc022969 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x40000000020000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3059 expref: 29 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.277433:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 1 to [0x2c0000401:0x8ee:0x0].0x0 since=37s v=(0 0) 00010000:00010000:1.0:1783520936.277438:0:20140:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ee:0x0].0x0 since=0s v=(1 0) 00000100:00100000:0.0:1783520936.277442:0:35282:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993301120 00010000:00010000:1.0:1783520936.277445:0:20140:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026c83 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.277450:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000cdcdc012/0xd5401cbabc0227f6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2eae expref: 29 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.277451:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000009c5b3794/0xd5401cbabc0227d3 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8e8:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e76 expref: 29 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.277499:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503a69500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993301376:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.277501:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.277501:0:20140:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000003a53433c/0xd5401cbabc022969 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x50000000030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3059 expref: 28 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.277511:0:35282:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45053 00010000:00010000:3.0:1783520936.277515:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e8:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.277530:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026c67 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.277540:0:20140:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000003a53433c/0xd5401cbabc022969 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x70000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3059 expref: 28 pid: 35276 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.277543:0:35282:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943507ae5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+979:3628:x1870152993301120:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.277551:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1780 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.277571:0:20140:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943505875f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993301504:192.168.204.33@tcp:104: 00002000:00080000:0.0:1783520936.277573:0:35282:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2285 count 1 00010000:00010000:0.0:1783520936.277598:0:35282:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5c4c3ff/0xd5401cbabc026c8a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8ed:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.277609:0:20140:0:(client.c:2702:ptlrpc_set_wait()) set ffff943512470a80 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.277618:0:35282:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000d3c1701c/0xd5401cbabc0227ef lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ebc expref: 28 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.277629:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ed:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.277633:0:35282:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ed:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.277638:0:35282:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5c4c3ff/0xd5401cbabc026c8a lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.277640:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026c67 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.277673:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.277694:0:35282:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000d3c1701c/0xd5401cbabc0227ef lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ebc expref: 28 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.277695:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:1.0:1783520936.277701:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002492672 00000100:00100000:1.0:1783520936.277711:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002492672 00010000:00010000:0.0:1783520936.277721:0:35282:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000d3c1701c/0xd5401cbabc0227ef lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ebc expref: 29 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.277740:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000002806103e/0xd5401cbabc0227e1 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e92 expref: 29 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.277741:0:35282:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435166c8380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993301632:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.277788:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36639 00000100:00100000:1.0:1783520936.277803:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b21f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+29:2389:x1870153002492672:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.277816:0:35282:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80900 going to sleep for 11 seconds 00000100:00100000:3.0:1783520936.277818:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517333480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+29:2388:x1870153002492544:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 653us (1238us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.277818:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.277837:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000002806103e/0xd5401cbabc0227e1 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8ea:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e92 expref: 29 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.277838:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36638 00010000:00010000:1.0:1783520936.277864:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000002806103e/0xd5401cbabc0227e1 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ea:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e92 expref: 28 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.277887:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc0000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993300864:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.277888:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ea:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.277892:0:12734:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026c67 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.277905:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ea:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.277915:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026c6e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ea:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.277917:0:12734:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026c67 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.277923:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026c75 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.277953:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026c6e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8ea:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.277982:0:12734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026c67 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:3.0:1783520936.277985:0:8409:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2281 count 1 00010000:00010000:3.0:1783520936.278012:0:8409:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c91 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8e9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.278025:0:12734:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026c67 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.278036:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.278044:0:12734:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.278056:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.278061:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026c67 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.278068:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.278072:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026c67 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.278073:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b21f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+28:2389:x1870153002492672:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 274us (410us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.278088:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026c67 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.278090:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36639 00010000:00010000:2.0:1783520936.278115:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026c67 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.278117:0:8409:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000071be8177/0xd5401cbabc0227da lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e84 expref: 27 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.278134:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000005120a6/0xd5401cbabc026c67 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 12734 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.278136:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e9:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.278145:0:8409:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.278151:0:8409:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c91 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.278161:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.278163:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.278165:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.278181:0:8409:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000071be8177/0xd5401cbabc0227da lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e84 expref: 27 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.278193:0:8409:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000071be8177/0xd5401cbabc0227da lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e84 expref: 27 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.278206:0:8409:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503370e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993301760:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.278249:0:8409:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.278262:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.278303:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350794df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+980:3625:x1870152993299968:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3873us (4002us total) trans 4294976548 rc 0/0 00010000:00010000:3.0:1783520936.278316:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026c52 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.278316:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45047 00000004:00080000:1.0:1783520936.278321:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503a6a300/2, rc 0, transno 4294976548 00000100:00100000:1.0:1783520936.278339:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503a6a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993299968:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.278404:0:35275:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000002806103e/0xd5401cbabc0227e1 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e92 expref: 27 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.278426:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943505874e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993300736:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.278432:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002492800 00000100:00100000:0.0:1783520936.278440:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002492800 00000100:00100000:1.0:1783520936.278448:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943505877800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993301888:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.278455:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026c6e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.278468:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026c52 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.278473:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026c6e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.278477:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a1d03729/0xd5401cbabc0227e8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ea0 expref: 27 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.278506:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.278512:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026c52 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.278541:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026c6e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.278543:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026c52 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.278567:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026c6e lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.278570:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36640 00010000:00010000:3.0:1783520936.278582:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.278585:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505875c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+27:2387:x1870153002492800:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.278595:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026c6e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.278603:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.278611:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a1d03729/0xd5401cbabc0227e8 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8eb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ea0 expref: 27 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.278618:0:7895:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.278628:0:7895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026c52 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.278629:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a1d03729/0xd5401cbabc0227e8 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8eb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ea0 expref: 27 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.278638:0:20140:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943505875f80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_005:lustre-OST0001_UUID:20140:1870152993301504:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.278668:0:7895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:1.0:1783520936.278668:0:20140:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026c83 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.278675:0:7895:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026c52 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.278687:0:7895:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026c52 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.278696:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a1d03729/0xd5401cbabc0227e8 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8eb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ea0 expref: 27 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.278707:0:7895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026c52 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.278719:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8eb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.278724:0:7895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ccb622bb/0xd5401cbabc026c52 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.278726:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8eb:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.278739:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026c75 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.278752:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026c75 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.278756:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.278761:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.278767:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:1.0:1783520936.278779:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff9435118b0700/2, rc 0, transno 4294976549 00000100:00100000:1.0:1783520936.278795:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993299072:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.278798:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.278813:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435032e8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+980:3627:x1870152993299072:12345-0@lo:6:osp-syn-1-0.0 Request processed in 5718us (5832us total) trans 4294976549 rc 0/0 00000100:00100000:3.0:1783520936.278826:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45044 00000100:00100000:2.0:1783520936.278836:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503a69500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993301376:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.278837:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1783520936.278851:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943505875f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993302016:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.278853:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993301888 00000100:00100000:0.0:1783520936.278855:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505875c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+26:2387:x1870153002492800:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 263us (587us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.278860:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026c7c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.278861:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993301888 00000100:00100000:0.0:1783520936.278873:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36640 00000100:00100000:1.0:1783520936.278882:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.278900:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993302016 00000100:00100000:3.0:1783520936.278905:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993302016 00000100:00100000:2.0:1783520936.278927:0:35276:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45054 00010000:00010000:0.0:1783520936.278933:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026c75 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.278945:0:35276:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943505874e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+981:3625:x1870152993301888:12345-0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.278954:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45055 00010000:00010000:0.0:1783520936.278970:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026c75 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.278972:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e0f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+981:3625:x1870152993302016:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.278976:0:35276:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2287 count 1 00010000:00010000:0.0:1783520936.279000:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026c75 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.279001:0:20415:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2288 count 1 00010000:00010000:2.0:1783520936.279004:0:35276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026c98 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8ef:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.279017:0:20415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026c9f lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8f0:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.279058:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026c75 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.279066:0:20415:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000aab6ea30/0xd5401cbabc02280b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ee6 expref: 25 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.279067:0:35276:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000022c7c6bc/0xd5401cbabc022804 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ed8 expref: 25 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.279100:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f0:0x0].0x0 since=38s v=(0 0) 00010000:00010000:0.0:1783520936.279103:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.279104:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ef:0x0].0x0 since=38s v=(0 0) 00010000:00010000:1.0:1783520936.279105:0:20415:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f0:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.279115:0:20415:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026c9f lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.279116:0:35276:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ef:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.279120:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.279130:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026c75 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.279130:0:35276:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026c98 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.279144:0:20415:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000aab6ea30/0xd5401cbabc02280b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ee6 expref: 25 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.279150:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeb400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.279156:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026c75 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.279161:0:20415:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000aab6ea30/0xd5401cbabc02280b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ee6 expref: 25 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.279171:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026c75 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.279177:0:20415:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943509174a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993302144:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.279178:0:35276:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000022c7c6bc/0xd5401cbabc022804 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ed8 expref: 25 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.279195:0:35276:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000022c7c6bc/0xd5401cbabc022804 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ed8 expref: 25 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.279212:0:35276:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b23b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993302272:192.168.204.33@tcp:104: 00000100:00100000:1.0:1783520936.279218:0:20415:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f000 going to sleep for 11 seconds 00010000:00010000:3.0:1783520936.279227:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026c75 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.279236:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000191ee13a/0xd5401cbabc026c75 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.279249:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.279253:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.279255:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.279257:0:35276:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1780 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.279268:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.279294:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435038ed180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+982:3626:x1870152993300608:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2995us (3140us total) trans 4294976550 rc 0/0 00000100:00100000:3.0:1783520936.279295:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002493056 00000100:00100000:3.0:1783520936.279305:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002493056 00000100:00100000:0.0:1783520936.279309:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45050 00010000:00010000:3.0:1783520936.279324:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000cdcdc012/0xd5401cbabc0227f6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2eae expref: 26 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.279386:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36641 00000100:00100000:3.0:1783520936.279392:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943509175880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+26:2387:x1870153002493056:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.279400:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.279405:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000cdcdc012/0xd5401cbabc0227f6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ec:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2eae expref: 26 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:0.0:1783520936.279407:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351e11dc00/2, rc 0, transno 4294976550 00010000:00010000:3.0:1783520936.279425:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000cdcdc012/0xd5401cbabc0227f6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8ec:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2eae expref: 26 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.279427:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e11dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993300608:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.279457:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cdcdc012/0xd5401cbabc0227f6 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8ec:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2eae expref: 26 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.279477:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ec:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.279481:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ec:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.279497:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026c7c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.279499:0:35282:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435166c8380 pname:cluuid:pid:xid:nid:opc:job ll_ost00_010:lustre-OST0001_UUID:35282:1870152993301632:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.279500:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.279514:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943514dd7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993302400:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.279527:0:35282:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000f5c4c3ff/0xd5401cbabc026c8a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.279543:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.279549:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026c6e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.279559:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1783520936.279569:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbaa00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.279575:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026c6e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.279586:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026c6e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.279607:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026c6e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.279613:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026c7c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.279616:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000258eb36f/0xd5401cbabc026c6e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.279623:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517244e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+982:3628:x1870152993300480:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3549us (4319us total) trans 4294976551 rc 0/0 00000100:00100000:1.0:1783520936.279636:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:2.0:1783520936.279662:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45049 00010000:00010000:0.0:1783520936.279675:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.279677:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.279683:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.279689:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.279695:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993302400 00000100:00100000:2.0:1783520936.279703:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993302400 00000100:00100000:3.0:1783520936.279711:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943509175880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+25:2387:x1870153002493056:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 317us (441us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.279721:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36641 00000100:00100000:1.0:1783520936.279730:0:3620:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000004:00080000:0.0:1783520936.279733:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943507acf100/2, rc 0, transno 4294976551 00000100:00100000:3.0:1783520936.279748:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002492928 00000100:00100000:2.0:1783520936.279748:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45056 00000100:00100000:3.0:1783520936.279753:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002492928 00000100:00100000:0.0:1783520936.279756:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943507acf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993300480:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.279773:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c553100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+982:3627:x1870152993302400:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.279783:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000003a53433c/0xd5401cbabc022969 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3059 expref: 25 pid: 35276 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:2.0:1783520936.279790:0:35275:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2289 count 1 00010000:00010000:2.0:1783520936.279808:0:35275:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026ca6 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8f1:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.279814:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943638995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993302528:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.279816:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002493184 00000100:00100000:3.0:1783520936.279829:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002493184 00000100:00100000:1.0:1783520936.279837:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36642 00010000:00010000:2.0:1783520936.279838:0:35275:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000020311c0/0xd5401cbabc022812 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ef4 expref: 26 pid: 178496 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.279850:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f1:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.279854:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000d3c1701c/0xd5401cbabc0227ef lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ebc expref: 26 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.279855:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350c550000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+26:2387:x1870153002492928:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1783520936.279857:0:35275:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f1:0x0].0x0 since=0s v=(0 0) 00000100:00100000:0.0:1783520936.279872:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.279877:0:35275:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026ca6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.279915:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36643 00010000:00010000:2.0:1783520936.279923:0:35275:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000020311c0/0xd5401cbabc022812 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ef4 expref: 26 pid: 178496 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.279929:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943638997800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+26:2387:x1870153002493184:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1783520936.279938:0:9151:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993302528 00000100:00100000:0.0:1783520936.279943:0:9151:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993302528 00010000:00010000:3.0:1783520936.279945:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.279945:0:35275:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000020311c0/0xd5401cbabc022812 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ef4 expref: 26 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.279953:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:3.0:1783520936.279955:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000d3c1701c/0xd5401cbabc0227ef lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ed:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ebc expref: 26 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.279965:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000003a53433c/0xd5401cbabc022969 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x8ee:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x60000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3059 expref: 26 pid: 35276 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.279972:0:35275:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b23480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993302656:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.279975:0:9151:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45057 00010000:00010000:3.0:1783520936.279983:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000d3c1701c/0xd5401cbabc0227ef lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8ed:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ebc expref: 26 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.279995:0:9151:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435166c8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+983:3628:x1870152993302528:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.280007:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000003a53433c/0xd5401cbabc022969 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8ee:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x50000400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3059 expref: 26 pid: 35276 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00080000:0.0:1783520936.280019:0:9151:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2290 count 1 00010000:00010000:1.0:1783520936.280032:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000003a53433c/0xd5401cbabc022969 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8ee:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 16777216->16781311) gid 0 flags: 0x44801400030020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3059 expref: 26 pid: 35276 timeout: 4231 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.280038:0:35275:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1a80 going to sleep for 11 seconds 00010000:00010000:1.0:1783520936.280047:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ee:0x0].0x0 since=0s v=(1 0) 00010000:00010000:0.0:1783520936.280048:0:9151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cd47ee70/0xd5401cbabc026cad lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8f2:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.280051:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ee:0x0].0x0 since=0s v=(1 0) 00010000:00010000:2.0:1783520936.280061:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026c7c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.280064:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026c83 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.280107:0:9151:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 000000007034d24f/0xd5401cbabc022819 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f02 expref: 25 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.280109:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026c83 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.280112:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026c7c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.280127:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000d3c1701c/0xd5401cbabc0227ef lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8ed:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ebc expref: 25 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.280173:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f2:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.280176:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026c7c lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.280179:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ed:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.280181:0:9151:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.280185:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:3.0:1783520936.280196:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ed:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.280200:0:9151:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cd47ee70/0xd5401cbabc026cad lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.280212:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000f5c4c3ff/0xd5401cbabc026c8a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.280214:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c550000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+24:2387:x1870153002492928:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 358us (586us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.280217:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026c7c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.280259:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000f5c4c3ff/0xd5401cbabc026c8a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.280264:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36642 00010000:00010000:2.0:1783520936.280268:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1783520936.280279:0:9151:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 000000007034d24f/0xd5401cbabc022819 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f02 expref: 23 pid: 8410 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.280281:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.280295:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026c7c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.280296:0:20140:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026c83 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.280316:0:9151:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 000000007034d24f/0xd5401cbabc022819 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f02 expref: 23 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.280322:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.280333:0:20140:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026c83 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.280354:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943638997800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+23:2387:x1870153002493184:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 423us (618us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.280406:0:20140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026c83 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.280424:0:20140:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026c83 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.280426:0:9151:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435166c8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993302784:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.280435:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184c00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.280446:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026c7c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.280460:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026c7c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.280460:0:20140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520936.280474:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36643 00010000:00010000:2.0:1783520936.280501:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026c7c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.280512:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000c1f24f65/0xd5401cbabc026c7c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.280520:0:9151:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80900 going to sleep for 11 seconds 00010000:00010000:2.0:1783520936.280546:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.280548:0:35282:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000f5c4c3ff/0xd5401cbabc026c8a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.280554:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026c83 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.280564:0:8409:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503370e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_001:lustre-OST0001_UUID:8409:1870152993301760:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.280595:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.280595:0:35282:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000f5c4c3ff/0xd5401cbabc026c8a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.280596:0:8409:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c91 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.280607:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026c83 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.280607:0:20140:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350793bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+984:3628:x1870152993301248:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3298us (3418us total) trans 4294976552 rc 0/0 00010000:00010000:0.0:1783520936.280647:0:35282:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000f5c4c3ff/0xd5401cbabc026c8a lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.280656:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026c83 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.280658:0:20140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45052 00010000:00010000:0.0:1783520936.280691:0:35282:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000f5c4c3ff/0xd5401cbabc026c8a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.280697:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026c83 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.280717:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007c1d6005/0xd5401cbabc026c83 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.280746:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.280751:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.280756:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520936.280762:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503372300/2, rc 0, transno 4294976552 00010000:00010000:2.0:1783520936.280769:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.280785:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503372300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993301248:0@lo:6:osp-syn-1-0.0 00010000:00010000:0.0:1783520936.280819:0:35282:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1783520936.280832:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943506228a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993302912:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.280833:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.280843:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000f5c4c3ff/0xd5401cbabc026c8a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.280851:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbebc00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:3.0:1783520936.280856:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5c4c3ff/0xd5401cbabc026c8a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.280866:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000f5c4c3ff/0xd5401cbabc026c8a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.280882:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1783520936.280884:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000f5c4c3ff/0xd5401cbabc026c8a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.280886:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350793ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+984:3626:x1870152993300992:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3715us (3820us total) trans 4294976553 rc 0/0 00000004:00080000:0.0:1783520936.280895:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943513139c00/2, rc 0, transno 4294976553 00000100:00100000:2.0:1783520936.280906:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45051 00010000:00010000:3.0:1783520936.280911:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000f5c4c3ff/0xd5401cbabc026c8a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35282 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.280917:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513139c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993300992:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.280928:0:35274:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993302912 00000100:00100000:2.0:1783520936.280934:0:35274:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993302912 00010000:00010000:3.0:1783520936.280946:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.280950:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.280953:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1783520936.280973:0:35274:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45058 00000100:00100000:0.0:1783520936.280975:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943502c9a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993303040:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.280985:0:35274:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350622b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+984:3625:x1870152993302912:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:2.0:1783520936.281003:0:35274:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2291 count 1 00000100:00100000:1.0:1783520936.281003:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.281023:0:35274:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026cb4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8f3:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.281035:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1783520936.281050:0:35274:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000b7248819/0xd5401cbabc022820 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f10 expref: 22 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.281059:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f3:0x0].0x0 since=38s v=(0 0) 00010000:00010000:2.0:1783520936.281065:0:35274:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.281070:0:35274:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026cb4 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.281078:0:8410:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993303040 00000100:00100000:3.0:1783520936.281086:0:8410:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993303040 00000100:00100000:0.0:1783520936.281110:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002493312 00010000:00010000:2.0:1783520936.281111:0:35274:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000b7248819/0xd5401cbabc022820 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f10 expref: 22 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.281120:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002493312 00010000:00010000:2.0:1783520936.281138:0:35274:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000b7248819/0xd5401cbabc022820 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f10 expref: 23 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.281158:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:0.0:1783520936.281159:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000071be8177/0xd5401cbabc0227da lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e84 expref: 23 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.281160:0:35274:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503a69500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993303168:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.281177:0:8410:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45059 00000100:00100000:1.0:1783520936.281192:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.281194:0:8410:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435166cb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+985:3626:x1870152993303040:12345-0@lo:6:osp-syn-1-0.0 00002000:00080000:3.0:1783520936.281211:0:8410:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2292 count 1 00000100:00100000:0.0:1783520936.281220:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002493440 00000100:00100000:2.0:1783520936.281221:0:35274:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c5b1300 going to sleep for 11 seconds 00000100:00100000:0.0:1783520936.281226:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002493440 00010000:00010000:3.0:1783520936.281227:0:8410:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026cbb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8f4:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.281245:0:8410:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 0000000061e31309/0xd5401cbabc022827 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f1e expref: 24 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.281246:0:17759:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002493568 00010000:00010000:0.0:1783520936.281255:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000aab6ea30/0xd5401cbabc02280b lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ee6 expref: 24 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.281257:0:17759:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002493568 00010000:00010000:3.0:1783520936.281258:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f4:0x0].0x0 since=38s v=(0 0) 00010000:00010000:3.0:1783520936.281277:0:8410:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f4:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.281278:0:6479:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36644 00010000:00010000:2.0:1783520936.281287:0:17759:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000022c7c6bc/0xd5401cbabc022804 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ed8 expref: 25 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.281302:0:8410:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026cbb lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.281302:0:6479:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357d850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+25:2387:x1870153002493312:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.281333:0:6479:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.281344:0:6479:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000071be8177/0xd5401cbabc0227da lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8e9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e84 expref: 25 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.281389:0:6479:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000071be8177/0xd5401cbabc0227da lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8e9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e84 expref: 25 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.281392:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36645 00000100:00100000:0.0:1783520936.281407:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351664b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+25:2388:x1870153002493440:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.281412:0:8410:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 0000000061e31309/0xd5401cbabc022827 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f1e expref: 25 pid: 20140 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.281418:0:6479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000071be8177/0xd5401cbabc0227da lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8e9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2e84 expref: 25 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.281419:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.281433:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000aab6ea30/0xd5401cbabc02280b lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8f0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ee6 expref: 25 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.281444:0:8410:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 0000000061e31309/0xd5401cbabc022827 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f1e expref: 25 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.281446:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36646 00010000:00010000:1.0:1783520936.281458:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e9:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.281473:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351664bb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+24:2390:x1870153002493568:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.281478:0:6479:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8e9:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.281505:0:6479:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c91 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.281515:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.281515:0:6479:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c91 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.281517:0:8410:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503370e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993303296:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.281518:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000aab6ea30/0xd5401cbabc02280b lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8f0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ee6 expref: 24 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.281533:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000022c7c6bc/0xd5401cbabc022804 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8ef:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ed8 expref: 24 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.281578:0:6479:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.281582:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000022c7c6bc/0xd5401cbabc022804 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8ef:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ed8 expref: 24 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.281586:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f0:0x0].0x0 since=0s v=(0 0) 00000100:00100000:3.0:1783520936.281589:0:8410:0:(client.c:2702:ptlrpc_set_wait()) set ffff943514d3e9c0 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.281603:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f0:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.281606:0:6479:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357d850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_002:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+24:2387:x1870153002493312:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 313us (600us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.281614:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026c9f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.281616:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ef:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.281622:0:6479:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36644 00010000:00010000:2.0:1783520936.281637:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8ef:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.281644:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026c9f lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8f0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.281647:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026c98 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ef:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.281685:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026c98 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8ef:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.281697:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.281707:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:0.0:1783520936.281739:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351664b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+23:2388:x1870153002493440:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 334us (584us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.281750:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351664bb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+23:2390:x1870153002493568:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 283us (560us total) trans 0 rc 0/0 00010000:00010000:3.0:1783520936.281755:0:8409:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c91 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.281758:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36645 00000100:00100000:2.0:1783520936.281776:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36646 00010000:00010000:3.0:1783520936.281816:0:8409:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c91 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.281827:0:8409:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c91 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.281837:0:35276:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000022c7c6bc/0xd5401cbabc022804 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ed8 expref: 21 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.281853:0:35276:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b23b80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_009:lustre-OST0001_UUID:35276:1870152993302272:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.281858:0:20415:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000aab6ea30/0xd5401cbabc02280b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ee6 expref: 20 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.281874:0:35276:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026c98 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.281880:0:20415:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943509174a80 pname:cluuid:pid:xid:nid:opc:job ll_ost00_006:lustre-OST0001_UUID:20415:1870152993302144:192.168.204.33@tcp:104: 00010000:00010000:2.0:1783520936.281895:0:35276:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026c98 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.281904:0:35276:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026c98 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.281906:0:20415:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026c9f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.281919:0:35276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026c98 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.281922:0:20415:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026c9f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.281928:0:8409:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c91 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.281944:0:35282:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943507ae5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_010:lustre-MDT0000-mdtlov_UUID+986:3628:x1870152993301120:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4406us (4655us total) trans 4294976554 rc 0/0 00010000:00010000:1.0:1783520936.281947:0:20415:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026c9f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.281978:0:8409:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1783520936.281983:0:35282:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45053 00010000:00010000:2.0:1783520936.281997:0:35276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026c98 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.281998:0:20415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026c9f lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.282015:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.282029:0:20415:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026c9f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.282032:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c91 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.282035:0:35276:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.282055:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1783520936.282065:0:20415:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.282069:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026c98 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.282069:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf5a00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.282076:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c91 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.282104:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184000 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.282108:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c91 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.282112:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026c98 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.282131:0:8409:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351313a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+987:3627:x1870152993300096:12345-0@lo:6:osp-syn-1-0.0 Request processed in 6361us (7594us total) trans 4294976555 rc 0/0 00000100:00100000:1.0:1783520936.282141:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e0f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0000-mdtlov_UUID+987:3625:x1870152993302016:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3168us (3262us total) trans 4294976556 rc 0/0 00010000:00010000:0.0:1783520936.282149:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c91 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.282152:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026c98 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.282153:0:8409:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45048 00000100:00100000:1.0:1783520936.282169:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45055 00010000:00010000:0.0:1783520936.282191:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000de2c21c5/0xd5401cbabc026c91 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8409 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.282195:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943503a68a80/2, rc 0, transno 4294976555 00010000:00010000:2.0:1783520936.282201:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026c98 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000004:00080000:1.0:1783520936.282208:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943505875f80/2, rc 0, transno 4294976556 00000100:00100000:3.0:1783520936.282219:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503a68a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993300096:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.282229:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943505875f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993302016:0@lo:6:osp-syn-1-0.0 00010000:00010000:2.0:1783520936.282243:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000efd5ffe8/0xd5401cbabc026c98 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35276 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.282261:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1783520936.282267:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026c9f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.282270:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.282275:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.282280:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.282283:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bbb600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:0.0:1783520936.282289:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026c9f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.282291:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.282302:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026c9f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.282326:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026c9f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.282334:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a08d6f18/0xd5401cbabc026c9f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 20415 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.282381:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351664a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993303424:0@lo:6:osp-syn-1-0.0 00000100:00100000:1.0:1783520936.282412:0:9151:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435166c8700 pname:cluuid:pid:xid:nid:opc:job ll_ost00_003:lustre-OST0001_UUID:9151:1870152993302784:192.168.204.33@tcp:104: 00010000:00010000:0.0:1783520936.282419:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.282424:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.282430:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1783520936.282441:0:9151:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 00000000cd47ee70/0xd5401cbabc026cad lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.282444:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.282451:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943516649500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993303552:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.282506:0:3618:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:0.0:1783520936.282515:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.282543:0:6478:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002493696 00000100:00100000:2.0:1783520936.282549:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993303424 00000100:00100000:3.0:1783520936.282550:0:6478:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002493696 00000100:00100000:2.0:1783520936.282555:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993303424 00000004:00080000:0.0:1783520936.282569:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94350d166a00/2, rc 0, transno 4294976554 00000100:00100000:0.0:1783520936.282577:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350d166a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993301120:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.282593:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993303552 00000100:00100000:2.0:1783520936.282599:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993303552 00000100:00100000:1.0:1783520936.282605:0:8408:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45060 00000100:00100000:1.0:1783520936.282619:0:8408:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+987:3627:x1870152993303424:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.282626:0:6478:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000020311c0/0xd5401cbabc022812 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ef4 expref: 20 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00002000:00080000:1.0:1783520936.282632:0:8408:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2293 count 1 00000100:00100000:0.0:1783520936.282634:0:178496:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 45061 00010000:00010000:1.0:1783520936.282649:0:8408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026cc2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8f5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.282662:0:178496:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435166c8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+987:3628:x1870152993303552:12345-0@lo:6:osp-syn-1-0.0 00010000:00010000:1.0:1783520936.282670:0:8408:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000a1415a93/0xd5401cbabc02282e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f2c expref: 20 pid: 178496 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.282677:0:6478:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36647 00000100:00100000:2.0:1783520936.282679:0:35274:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503a69500 pname:cluuid:pid:xid:nid:opc:job ll_ost00_007:lustre-OST0001_UUID:35274:1870152993303168:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.282690:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f5:0x0].0x0 since=38s v=(0 0) 00000100:00100000:3.0:1783520936.282696:0:6478:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+20:2387:x1870153002493696:12345-192.168.204.33@tcp:103:ldlm_bl.0 00002000:00080000:0.0:1783520936.282696:0:178496:0:(ofd_dev.c:1862:ofd_destroy_hdl()) lustre-OST0001: Destroy object 0x2c0000401:2294 count 1 00010000:00010000:1.0:1783520936.282697:0:8408:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.282701:0:35274:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026cb4 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.282714:0:8408:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026cc2 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.282719:0:6478:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.282726:0:178496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026cc9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8f6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x40000000000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.282748:0:178496:0:(ldlm_lock.c:687:ldlm_add_bl_work_item()) ### lock incompatible; sending blocking AST. ns: filter-lustre-OST0001_UUID lock: 00000000ddf44b9d/0xd5401cbabc024f1e lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x8f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40000000000000 nid: 192.168.204.33@tcp remote: 0xc275876b859f3450 expref: 20 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.282749:0:8408:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000a1415a93/0xd5401cbabc02282e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f2c expref: 20 pid: 178496 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.282789:0:6478:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000020311c0/0xd5401cbabc022812 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8f1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ef4 expref: 20 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.282793:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f6:0x0].0x0 since=3s v=(0 0) 00010000:00010000:1.0:1783520936.282795:0:8408:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000a1415a93/0xd5401cbabc02282e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f2c expref: 20 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.282811:0:178496:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f6:0x0].0x0 since=0s v=(0 0) 00000100:00100000:2.0:1783520936.282827:0:35276:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943505874e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_009:lustre-MDT0000-mdtlov_UUID+988:3625:x1870152993301888:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3885us (4338us total) trans 4294976557 rc 0/0 00010000:00010000:3.0:1783520936.282831:0:6478:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000020311c0/0xd5401cbabc022812 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8f1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ef4 expref: 20 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.282833:0:178496:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026cc9 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x8f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.282840:0:8408:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351664aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993303680:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.282852:0:35276:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45054 00010000:00010000:0.0:1783520936.282890:0:178496:0:(ldlm_lockd.c:963:ldlm_server_blocking_ast()) ### server preparing blocking AST ns: filter-lustre-OST0001_UUID lock: 00000000ddf44b9d/0xd5401cbabc024f1e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000000010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3450 expref: 20 pid: 8408 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:2.0:1783520936.282904:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943505877800/2, rc 0, transno 4294976557 00000100:00100000:1.0:1783520936.282909:0:8408:0:(client.c:2702:ptlrpc_set_wait()) set ffff94350c01f000 going to sleep for 11 seconds 00010000:00010000:0.0:1783520936.282909:0:178496:0:(ldlm_lockd.c:495:ldlm_add_waiting_lock()) ### adding to wait list(timeout: 100, AT: on) ns: filter-lustre-OST0001_UUID lock: 00000000ddf44b9d/0xd5401cbabc024f1e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x70000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3450 expref: 20 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.282919:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943505877800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993301888:0@lo:6:osp-syn-1-0.0 00010000:00010000:3.0:1783520936.282929:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f1:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.282945:0:6478:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f1:0x0].0x0 since=0s v=(0 0) 00000100:00100000:1.0:1783520936.282949:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.282959:0:35275:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000020311c0/0xd5401cbabc022812 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8f1:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2ef4 expref: 20 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.282962:0:6478:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026ca6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f1:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.282968:0:178496:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513bc0000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993303808:192.168.204.33@tcp:104: 00000100:00100000:2.0:1783520936.282978:0:35275:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b23480 pname:cluuid:pid:xid:nid:opc:job ll_ost00_008:lustre-OST0001_UUID:35275:1870152993302656:192.168.204.33@tcp:104: 00010000:00010000:3.0:1783520936.282992:0:6478:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026ca6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.283006:0:35275:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026ca6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.283014:0:6478:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.283015:0:35275:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026ca6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.283024:0:35275:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026ca6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.283035:0:35275:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026ca6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.283040:0:35275:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026ca6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.283045:0:178496:0:(client.c:2702:ptlrpc_set_wait()) set ffff943511c80cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1783520936.283053:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:2.0:1783520936.283059:0:35275:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1783520936.283090:0:6478:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514ff2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_001:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+19:2387:x1870153002493696:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 360us (547us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.283090:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002493952 00000100:00100000:1.0:1783520936.283098:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002493952 00000100:00100000:3.0:1783520936.283105:0:6478:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36647 00010000:00010000:1.0:1783520936.283122:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000b7248819/0xd5401cbabc022820 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f10 expref: 19 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.283135:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.283142:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026ca6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.283151:0:178913:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002493824 00010000:00010000:3.0:1783520936.283152:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512185e00 already unused, calling callback (ffffffffc0bf9e90) 00000100:00100000:1.0:1783520936.283155:0:178913:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002493824 00010000:00010000:3.0:1783520936.283160:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026ca6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.283168:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350c553100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0000-mdtlov_UUID+988:3627:x1870152993302400:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3403us (3613us total) trans 4294976558 rc 0/0 00010000:00010000:1.0:1783520936.283170:0:178913:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 000000007034d24f/0xd5401cbabc022819 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f02 expref: 20 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.283180:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026ca6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.283188:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45056 00010000:00010000:3.0:1783520936.283210:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026ca6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.283218:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000092ba5ab2/0xd5401cbabc026ca6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35275 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.283224:0:178913:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36648 00000100:00100000:2.0:1783520936.283236:0:17759:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36649 00000100:00100000:1.0:1783520936.283241:0:178913:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517333480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+20:2390:x1870153002493952:12345-192.168.204.33@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1783520936.283253:0:17759:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b23480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+20:2389:x1870153002493824:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1783520936.283256:0:178913:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.283262:0:17759:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:1.0:1783520936.283264:0:178913:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000b7248819/0xd5401cbabc022820 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8f3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f10 expref: 20 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.283271:0:17759:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 000000007034d24f/0xd5401cbabc022819 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8f2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f02 expref: 20 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.283287:0:17759:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 000000007034d24f/0xd5401cbabc022819 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8f2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f02 expref: 20 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.283287:0:178913:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000b7248819/0xd5401cbabc022820 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8f3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f10 expref: 20 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.283293:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.283298:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.283308:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1783520936.283317:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00010000:00010000:1.0:1783520936.283340:0:178913:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000b7248819/0xd5401cbabc022820 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8f3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f10 expref: 20 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000004:00080000:3.0:1783520936.283342:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943514dd7100/2, rc 0, transno 4294976558 00010000:00010000:2.0:1783520936.283343:0:17759:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000007034d24f/0xd5401cbabc022819 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8f2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f02 expref: 20 pid: 8410 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.283353:0:8424:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002494080 00000100:00100000:3.0:1783520936.283355:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943514dd7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993302400:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.283397:0:8424:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002494080 00010000:00010000:2.0:1783520936.283410:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.283412:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.283421:0:8424:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 0000000061e31309/0xd5401cbabc022827 lrc: 5/0,0 mode: PR/PR res: [0x2c0000401:0x8f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f1e expref: 19 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.283423:0:17759:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f2:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.283424:0:178913:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f3:0x0].0x0 since=0s v=(0 0) 00010000:00010000:1.0:1783520936.283457:0:178913:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026cb4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.283458:0:17759:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 00000000cd47ee70/0xd5401cbabc026cad lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.283462:0:8410:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503370e00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_002:lustre-OST0001_UUID:8410:1870152993303296:192.168.204.33@tcp:104: 00000100:00100000:0.0:1783520936.283479:0:8424:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36650 00010000:00010000:3.0:1783520936.283502:0:8410:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026cbb lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.283505:0:178913:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026cb4 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.283507:0:17759:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 00000000cd47ee70/0xd5401cbabc026cad lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.283517:0:8424:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e3b0a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+19:2388:x1870153002494080:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1783520936.283545:0:8424:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:0.0:1783520936.283554:0:8424:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 0000000061e31309/0xd5401cbabc022827 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8f4:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f1e expref: 19 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.283568:0:17759:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:0.0:1783520936.283588:0:8424:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 0000000061e31309/0xd5401cbabc022827 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8f4:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f1e expref: 19 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.283596:0:178913:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:2.0:1783520936.283615:0:17759:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b23480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_005:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+19:2389:x1870153002493824:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 367us (565us total) trans 0 rc 0/0 00010000:00010000:0.0:1783520936.283625:0:8424:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000061e31309/0xd5401cbabc022827 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8f4:0x0].0x0 rrc: 5 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f1e expref: 19 pid: 20140 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.283637:0:17759:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36649 00000100:00100000:1.0:1783520936.283640:0:178913:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517333480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_006:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+19:2390:x1870153002493952:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 394us (688us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.283681:0:178913:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36648 00010000:00010000:0.0:1783520936.283683:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.283685:0:35274:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026cb4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.283695:0:8424:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f4:0x0].0x0 since=0s v=(0 0) 00010000:00010000:0.0:1783520936.283711:0:8424:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026cbb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.283713:0:35274:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026cb4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.283757:0:8424:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026cbb lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.283759:0:35274:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026cb4 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.283796:0:35274:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026cb4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.283799:0:9151:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 00000000cd47ee70/0xd5401cbabc026cad lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.283810:0:8424:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:1.0:1783520936.283820:0:9151:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 00000000cd47ee70/0xd5401cbabc026cad lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.283825:0:35274:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.283832:0:8410:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026cbb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.283833:0:9151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 00000000cd47ee70/0xd5401cbabc026cad lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.283839:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1783520936.283852:0:8424:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e3b0a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_003:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+16:2388:x1870153002494080:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 339us (545us total) trans 0 rc 0/0 00010000:00010000:2.0:1783520936.283853:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026cb4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.283862:0:9151:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 00000000cd47ee70/0xd5401cbabc026cad lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1783520936.283883:0:8424:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36650 00010000:00010000:2.0:1783520936.283905:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943512184200 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.283912:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026cb4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.283912:0:9151:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.283925:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026cb4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.283941:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026cb4 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.283956:0:8410:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026cbb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.283958:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000044471100/0xd5401cbabc026cb4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 35274 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.283981:0:8410:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026cbb lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.283992:0:8410:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026cbb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.284003:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.284009:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 00000000cd47ee70/0xd5401cbabc026cad lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.284018:0:8410:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.284018:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbea400 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.284025:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 00000000cd47ee70/0xd5401cbabc026cad lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.284031:0:9495:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.284040:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 00000000cd47ee70/0xd5401cbabc026cad lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.284040:0:9151:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435166c8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+988:3628:x1870152993302528:12345-0@lo:6:osp-syn-1-0.0 Request processed in 4047us (4177us total) trans 4294976559 rc 0/0 00010000:00010000:0.0:1783520936.284044:0:9495:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026cbb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.284079:0:9151:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45057 00010000:00010000:0.0:1783520936.284087:0:9495:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbf4800 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.284092:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 00000000cd47ee70/0xd5401cbabc026cad lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.284092:0:9495:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026cbb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.284115:0:8410:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435166cb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+988:3626:x1870152993303040:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2919us (3081us total) trans 4294976560 rc 0/0 00000100:00100000:1.0:1783520936.284131:0:8408:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351664aa00 pname:cluuid:pid:xid:nid:opc:job ll_ost00_000:lustre-OST0001_UUID:8408:1870152993303680:192.168.204.33@tcp:104: 00000100:00100000:3.0:1783520936.284150:0:8410:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45059 00010000:00010000:2.0:1783520936.284152:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000cd47ee70/0xd5401cbabc026cad lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 9151 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.284153:0:9495:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026cbb lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.284164:0:8408:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026cc2 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.284207:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.284215:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.284220:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.284231:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.284235:0:9495:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026cbb lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.284246:0:9495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 0000000023f15860/0xd5401cbabc026cbb lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8410 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1783520936.284270:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.284274:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1783520936.284280:0:9495:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:0.0:1783520936.284318:0:3626:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943502c9a680/2, rc 0, transno 4294976560 00000100:00100000:0.0:1783520936.284328:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943502c9a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3626:1870152993303040:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.284336:0:35274:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350622b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_007:lustre-MDT0000-mdtlov_UUID+988:3625:x1870152993302912:12345-0@lo:6:osp-syn-1-0.0 Request processed in 3347us (3452us total) trans 4294976561 rc 0/0 00000100:00100000:2.0:1783520936.284351:0:35274:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45058 00000004:00080000:1.0:1783520936.284356:0:3625:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943506228a80/2, rc 0, transno 4294976561 00000100:00100000:1.0:1783520936.284388:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943506228a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993302912:0@lo:6:osp-syn-1-0.0 00000004:00080000:0.0:1783520936.284425:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943638995180/2, rc 0, transno 4294976559 00000100:00100000:0.0:1783520936.284433:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943638995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993302528:0@lo:6:osp-syn-1-0.0 00000100:00100000:0.0:1783520936.284478:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.284519:0:10310:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002494208 00000100:00100000:3.0:1783520936.284526:0:10310:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002494208 00010000:00010000:3.0:1783520936.284552:0:10310:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000a1415a93/0xd5401cbabc02282e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f2c expref: 16 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.284568:0:178496:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513bc0000 pname:cluuid:pid:xid:nid:opc:job ll_ost00_011:lustre-OST0001_UUID:178496:1870152993303808:192.168.204.33@tcp:104: 00010000:00010000:1.0:1783520936.284590:0:178496:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026cc9 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x8f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.284598:0:10310:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36651 00000100:00100000:0.0:1783520936.284610:0:3619:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.33@tcp (source: 12345-192.168.204.33@tcp) 00000100:00100000:3.0:1783520936.284619:0:10310:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94363357ec50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+16:2388:x1870153002494208:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.284631:0:10310:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00000100:00100000:2.0:1783520936.284633:0:6477:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870153002494336 00000100:00100000:2.0:1783520936.284639:0:6477:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870153002494336 00010000:00010000:2.0:1783520936.284658:0:6477:0:(ldlm_lockd.c:2683:ldlm_cancel_hpreq_check()) ### hpreq cancel/convert lock ns: filter-lustre-OST0001_UUID lock: 00000000ddf44b9d/0xd5401cbabc024f1e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3450 expref: 17 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.284678:0:10310:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000a1415a93/0xd5401cbabc02282e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8f5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f2c expref: 17 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.284689:0:6477:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.33@tcp, seq: 36652 00010000:00010000:3.0:1783520936.284694:0:10310:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000a1415a93/0xd5401cbabc02282e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8f5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f2c expref: 17 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1783520936.284704:0:6477:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350626ed80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+17:2388:x1870153002494336:12345-192.168.204.33@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1783520936.284715:0:10310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000a1415a93/0xd5401cbabc02282e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8f5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f2f2c expref: 17 pid: 178496 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.284715:0:6477:0:(ldlm_lockd.c:1747:ldlm_request_cancel()) ### server-side cancel handler START: 1 locks, starting at 0 00010000:00010000:2.0:1783520936.284724:0:6477:0:(ldlm_lockd.c:1796:ldlm_request_cancel()) ### server cancels blocked lock after 0s ns: filter-lustre-OST0001_UUID lock: 00000000ddf44b9d/0xd5401cbabc024f1e lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x8f6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x60000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3450 expref: 17 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.284732:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.284737:0:10310:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f5:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.284739:0:6477:0:(ldlm_lockd.c:568:ldlm_del_waiting_lock()) ### removed ns: filter-lustre-OST0001_UUID lock: 00000000ddf44b9d/0xd5401cbabc024f1e lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x8f6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50000400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3450 expref: 16 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.284743:0:10310:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026cc2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.284758:0:10310:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026cc2 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.284761:0:6477:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 00000000ddf44b9d/0xd5401cbabc024f1e lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x8f6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44801400010020 nid: 192.168.204.33@tcp remote: 0xc275876b859f3450 expref: 16 pid: 8408 timeout: 4231 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.284783:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f6:0x0].0x0 since=0s v=(0 0) 00010000:00010000:2.0:1783520936.284787:0:6477:0:(ldlm_extent.c:408:ldlm_update_contention()) add 0 to [0x2c0000401:0x8f6:0x0].0x0 since=0s v=(0 0) 00010000:00010000:3.0:1783520936.284791:0:10310:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00010000:00010000:2.0:1783520936.284792:0:6477:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026cc9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.284803:0:6477:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026cc9 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x8f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40090080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.284806:0:8408:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026cc2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.284819:0:8408:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026cc2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1783520936.284829:0:8408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026cc2 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.284830:0:6477:0:(ldlm_lockd.c:1808:ldlm_request_cancel()) ### server-side cancel handler END 00000100:00100000:3.0:1783520936.284837:0:10310:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94363357ec50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_004:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+15:2388:x1870153002494208:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 222us (356us total) trans 0 rc 0/0 00010000:00010000:1.0:1783520936.284841:0:8408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026cc2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1783520936.284851:0:10310:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36651 00000100:00100000:2.0:1783520936.284855:0:6477:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350626ed80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cn00_000:f13ce486-cf55-47e1-a3f1-9e9fbcf7c3ff+15:2388:x1870153002494336:12345-192.168.204.33@tcp:103:ldlm_bl.0 Request processed in 154us (261us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.284865:0:6477:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.33@tcp, seq: 36652 00010000:00010000:1.0:1783520936.284870:0:8408:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1783520936.284892:0:178496:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026cc9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.284900:0:178496:0:(ldlm_request.c:557:ldlm_cli_enqueue_local()) ### client-side local enqueue handler, new lock created ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026cc9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.284906:0:178496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026cc9 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x8f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010080000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.284911:0:178496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026cc9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x50010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.284916:0:6481:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1783520936.284922:0:178496:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1783520936.284924:0:6481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026cc2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.284931:0:6480:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1783520936.284937:0:6481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff943513bba600 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.284939:0:6480:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026cc9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.284945:0:6481:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026cc2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.284946:0:6480:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff94351cbeba00 already unused, calling callback (ffffffffc0bf9e90) 00010000:00010000:2.0:1783520936.284952:0:6480:0:(ldlm_request.c:378:ldlm_blocking_ast_nocheck()) ### already unused, calling ldlm_cli_cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026cc9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x8f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40010480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.284967:0:6480:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026cc9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.284969:0:6481:0:(ldlm_request.c:1416:ldlm_cli_cancel_local()) ### server-side local cancel ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026cc2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x8f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x40018480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.284978:0:8408:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+988:3627:x1870152993303424:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2362us (2538us total) trans 4294976562 rc 0/0 00010000:00010000:2.0:1783520936.284980:0:6480:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026cc9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1783520936.284996:0:8408:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45060 00010000:00010000:2.0:1783520936.284997:0:6480:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000006f28a372/0xd5401cbabc026cc9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 178496 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1783520936.285015:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.285017:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1783520936.285019:0:6480:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.285033:0:6481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026cc2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x8f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.285043:0:6481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: filter-lustre-OST0001_UUID lock: 000000001ce58c00/0xd5401cbabc026cc2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x8f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x44819480000000 nid: local remote: 0x0 expref: -99 pid: 8408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1783520936.285059:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.285061:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1783520936.285064:0:6481:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00080000:3.0:1783520936.285086:0:3627:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff94351664a300/2, rc 0, transno 4294976562 00000100:00100000:3.0:1783520936.285092:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351664a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993303424:0@lo:6:osp-syn-1-0.0 00000100:00100000:2.0:1783520936.285107:0:178496:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435166c8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_011:lustre-MDT0000-mdtlov_UUID+988:3628:x1870152993303552:12345-0@lo:6:osp-syn-1-0.0 Request processed in 2458us (2594us total) trans 4294976563 rc 0/0 00000100:00100000:2.0:1783520936.285113:0:178496:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 45061 00000004:00080000:0.0:1783520936.285152:0:3628:0:(osp_sync.c:622:osp_sync_interpret()) reply req ffff943516649500/2, rc 0, transno 4294976563 00000100:00100000:0.0:1783520936.285161:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943516649500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993303552:0@lo:6:osp-syn-1-0.0 00000100:00100000:3.0:1783520936.373538:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503370e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993304064:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1783520936.373538:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3626:1870152993304320:0@lo:400:kworker.0 00000100:00100000:0.0:1783520936.373615:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943503370380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3628:1870152993304576:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1783520936.373624:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.373631:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3627:1870152993304448:0@lo:400:kworker.0 00000100:00100000:0.0:1783520936.373657:0:6498:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993304320 00000100:00100000:3.0:1783520936.373665:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.373677:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:3626:1870152993304832:0@lo:400:kworker.0 00000100:00100000:0.0:1783520936.373677:0:6498:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993304320 00000100:00100000:2.0:1783520936.373703:0:6499:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993304064 00000100:00100000:3.0:1783520936.373706:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94350626dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:ea7ffe8e-f913-4c2c-b57a-a81a86292f4f:3625:1870152993304192:0@lo:400:kworker.0 00000100:00100000:2.0:1783520936.373710:0:6499:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993304064 00000100:00100000:1.0:1783520936.373722:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.373741:0:6498:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 22660 00000100:00100000:1.0:1783520936.373752:0:3626:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515476680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3626:1870152993305344:0@lo:400:kworker.0 00000100:00100000:3.0:1783520936.373759:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.373759:0:6499:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 22661 00000100:00100000:0.0:1783520936.373759:0:6498:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b21f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+7:3626:x1870152993304320:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1783520936.373775:0:6499:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503371c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3625:x1870152993304064:12345-0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1783520936.373785:0:3626:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00000024:2.0:1783520936.373813:0:6499:0:(mdt_handler.c:540:mdt_statfs()) blocks cached 0 granted 4292608 pending 0 free 1443401728 avail 1297162240 00000100:00100000:1.0:1783520936.373822:0:20415:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993305344 00000100:00100000:3.0:1783520936.373827:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943513b23b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993303936:0@lo:13:osp-pre-1-0.0 00000100:00100000:1.0:1783520936.373827:0:20415:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993305344 00000100:00100000:0.0:1783520936.373832:0:6498:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b21f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+7:3626:x1870152993304320:12345-0@lo:400:kworker.0 Request processed in 79us (208us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.373847:0:6498:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 22660 00000100:00100000:2.0:1783520936.373871:0:6499:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503371c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3625:x1870152993304064:12345-0@lo:41:osp-pre-1-0.0 Request processed in 98us (204us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.373880:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.373883:0:20415:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4760 00000100:00100000:2.0:1783520936.373885:0:6499:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 22661 00000100:00100000:1.0:1783520936.373899:0:20415:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513bc0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0001-mdtlov_UUID+5:3626:x1870152993305344:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1783520936.373903:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1783520936.373911:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0000_UUID:3625:1870152993304704:0@lo:400:kworker.0 00000100:00100000:2.0:1783520936.373915:0:35279:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993304832 00000100:00100000:2.0:1783520936.373920:0:35279:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993304832 00000100:00100000:0.0:1783520936.373935:0:8318:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993304576 00000100:00100000:0.0:1783520936.373940:0:8318:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993304576 00000100:00100000:1.0:1783520936.373958:0:20415:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513bc0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_006:lustre-MDT0001-mdtlov_UUID+5:3626:x1870152993305344:12345-0@lo:400:kworker.0 Request processed in 62us (169us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.373960:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.373965:0:35279:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 7079 00000100:00100000:0.0:1783520936.373975:0:8318:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 22662 00000100:00100000:1.0:1783520936.373976:0:20415:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4760 00000100:00100000:2.0:1783520936.373980:0:35279:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b23480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_010:lustre-MDT0001-lwp-OST0000_UUID+5:3626:x1870152993304832:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1783520936.373994:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517208e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3625:1870152993305216:0@lo:400:kworker.0 00000100:00100000:0.0:1783520936.373999:0:8318:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943503324700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3628:x1870152993304576:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1783520936.374024:0:8318:0:(mdt_handler.c:540:mdt_statfs()) blocks cached 0 granted 4194304 pending 0 free 1442652160 avail 1296412672 00000100:00100000:1.0:1783520936.374033:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515476680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3626:1870152993305344:0@lo:400:kworker.0 00000100:00100000:2.0:1783520936.374037:0:35279:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b23480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_010:lustre-MDT0001-lwp-OST0000_UUID+5:3626:x1870152993304832:12345-0@lo:400:kworker.0 Request processed in 60us (314us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.374043:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1783520936.374052:0:35279:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 7079 00000100:00100000:1.0:1783520936.374058:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3626:1870152993304320:0@lo:400:kworker.0 00000100:00100000:0.0:1783520936.374070:0:8318:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943503324700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3628:x1870152993304576:12345-0@lo:41:osp-pre-0-1.0 Request processed in 83us (169us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.374071:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff94351e006d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3625:1870152993305728:0@lo:400:kworker.0 00000100:00100000:2.0:1783520936.374071:0:35279:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993304704 00000100:00100000:2.0:1783520936.374078:0:35279:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993304704 00000100:00100000:0.0:1783520936.374086:0:8318:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 22662 00000100:00100000:1.0:1783520936.374095:0:3626:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:3626:1870152993304832:0@lo:400:kworker.0 00000100:00100000:3.0:1783520936.374114:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.374116:0:35284:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993305216 00000100:00100000:0.0:1783520936.374121:0:35284:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993305216 00000100:00100000:2.0:1783520936.374125:0:35279:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993305728 00000100:00100000:2.0:1783520936.374131:0:35279:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993305728 00000100:00100000:3.0:1783520936.374149:0:3625:0:(client.c:3112:ptlrpc_free_committed()) lustre-MDT0001-osp-MDT0000: committing for last_committed 4294985692 gen 2 00000004:00080000:3.0:1783520936.374155:0:3625:0:(osp_trans.c:1040:osp_request_commit_cb()) trans no 4294985692 committed transno 4294985692 00000100:00100000:1.0:1783520936.374155:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503370380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3628:1870152993304576:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1783520936.374160:0:35284:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 7080 00000100:00100000:0.0:1783520936.374173:0:35284:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e3b0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_013:lustre-MDT0000-lwp-OST0000_UUID+5:3625:x1870152993304704:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1783520936.374175:0:35279:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 7081 00000100:00100000:3.0:1783520936.374181:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943503370e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993304064:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1783520936.374186:0:17758:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 7082 00000100:00100000:2.0:1783520936.374191:0:35279:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350544e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_010:lustre-MDT0001-lwp-OST0001_UUID+5:3625:x1870152993305216:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1783520936.374200:0:17758:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943502250e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_006:lustre-MDT0000-lwp-MDT0000_UUID+5:3625:x1870152993305728:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1783520936.374218:0:31119:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993304192 00000100:00100000:3.0:1783520936.374223:0:31119:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993304192 00000100:00100000:0.0:1783520936.374234:0:35284:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e3b0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_013:lustre-MDT0000-lwp-OST0000_UUID+5:3625:x1870152993304704:12345-0@lo:400:kworker.0 Request processed in 64us (278us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.374244:0:35284:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 7080 00000100:00100000:2.0:1783520936.374249:0:35279:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350544e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_010:lustre-MDT0001-lwp-OST0001_UUID+5:3625:x1870152993305216:12345-0@lo:400:kworker.0 Request processed in 60us (207us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.374257:0:17758:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943502250e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_006:lustre-MDT0000-lwp-MDT0000_UUID+5:3625:x1870152993305728:12345-0@lo:400:kworker.0 Request processed in 59us (142us total) trans 0 rc 0/0 00000100:00100000:2.0:1783520936.374263:0:35279:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 7081 00000100:00100000:3.0:1783520936.374266:0:31119:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1214 00000100:00100000:1.0:1783520936.374274:0:17758:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 7082 00000100:00100000:3.0:1783520936.374278:0:31119:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94360b04f450 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:ea7ffe8e-f913-4c2c-b57a-a81a86292f4f+24:3625:x1870152993304192:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1783520936.374304:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943517209500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3628:1870152993305088:0@lo:400:kworker.0 00000100:00100000:0.0:1783520936.374316:0:8412:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993303936 00000100:00100000:3.0:1783520936.374319:0:31119:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94360b04f450 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0001:ea7ffe8e-f913-4c2c-b57a-a81a86292f4f+24:3625:x1870152993304192:12345-0@lo:400:kworker.0 Request processed in 42us (566us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.374320:0:8412:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993303936 00000100:00100000:3.0:1783520936.374330:0:31119:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1214 00000100:00100000:1.0:1783520936.374338:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.374350:0:8412:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3193 00000100:00100000:0.0:1783520936.374379:0:8412:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943517333480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0000-mdtlov_UUID+988:3625:x1870152993303936:12345-0@lo:13:osp-pre-1-0.0 00002000:00000024:0.0:1783520936.374391:0:8412:0:(ofd_obd.c:735:ofd_statfs()) blocks cached 0 granted 977893952 pending 360448 free 3782025216 avail 3550502912 00000100:00100000:1.0:1783520936.374394:0:3628:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515474380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993305600:0@lo:400:kworker.0 00000100:00100000:3.0:1783520936.374418:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517208e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3625:1870152993305216:0@lo:400:kworker.0 00000100:00100000:0.0:1783520936.374430:0:8412:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943517333480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0000-mdtlov_UUID+988:3625:x1870152993303936:12345-0@lo:13:osp-pre-1-0.0 Request processed in 74us (555us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.374439:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94351e006d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3625:1870152993305728:0@lo:400:kworker.0 00000100:00100000:1.0:1783520936.374442:0:3628:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.374442:0:8412:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3193 00000100:00100000:3.0:1783520936.374463:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff94350626dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:ea7ffe8e-f913-4c2c-b57a-a81a86292f4f:3625:1870152993304192:0@lo:400:kworker.0 00000100:00080000:0.0:1783520936.374464:0:6489:0:(pinger.c:500:ping_evictor_main()) evicting all exports of obd lustre-MDT0000 00000100:00080000:0.0:1783520936.374471:0:6489:0:(pinger.c:500:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1783520936.374475:0:6489:0:(pinger.c:500:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1783520936.374481:0:12734:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993305600 00000100:00100000:1.0:1783520936.374498:0:12734:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993305600 00000100:00100000:3.0:1783520936.374502:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943513b23b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993303936:0@lo:13:osp-pre-1-0.0 00000100:00100000:0.0:1783520936.374506:0:10322:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993305088 00000100:00100000:0.0:1783520936.374514:0:10322:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993305088 00000100:00100000:3.0:1783520936.374521:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0000_UUID:3625:1870152993304704:0@lo:400:kworker.0 00000100:00100000:1.0:1783520936.374538:0:12734:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4761 00000100:00100000:1.0:1783520936.374549:0:12734:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435121a5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+6:3628:x1870152993305600:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1783520936.374562:0:10322:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 7083 00000100:00100000:0.0:1783520936.374570:0:10322:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9435118b4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3628:x1870152993305088:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1783520936.374590:0:12734:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435121a5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+6:3628:x1870152993305600:12345-0@lo:400:kworker.0 Request processed in 42us (152us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.374599:0:12734:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4761 00000100:00100000:0.0:1783520936.374604:0:10322:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9435118b4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3628:x1870152993305088:12345-0@lo:400:kworker.0 Request processed in 34us (267us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.374612:0:10322:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 7083 00000100:00100000:3.0:1783520936.374635:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1783520936.374636:0:6489:0:(pinger.c:500:ping_evictor_main()) evicting all exports of obd lustre-MDT0000 00000100:00100000:1.0:1783520936.374643:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515474380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3628:1870152993305600:0@lo:400:kworker.0 00000100:00100000:3.0:1783520936.374670:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435118b6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3627:1870152993304960:0@lo:400:kworker.0 00000100:00100000:1.0:1783520936.374675:0:35278:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993304448 00000100:00100000:1.0:1783520936.374681:0:35278:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993304448 00000100:00100000:3.0:1783520936.374702:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.374716:0:35278:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 7084 00000100:00100000:1.0:1783520936.374725:0:35278:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943513b23b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_009:lustre-MDT0000-lwp-MDT0001_UUID+5:3627:x1870152993304448:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1783520936.374736:0:35275:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993304960 00000100:00100000:3.0:1783520936.374741:0:35275:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993304960 00000100:00100000:0.0:1783520936.374759:0:3628:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943517209500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3628:1870152993305088:0@lo:400:kworker.0 00000100:00100000:1.0:1783520936.374767:0:35278:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943513b23b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_009:lustre-MDT0000-lwp-MDT0001_UUID+5:3627:x1870152993304448:12345-0@lo:400:kworker.0 Request processed in 41us (132us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.374776:0:35278:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 7084 00000100:00100000:3.0:1783520936.374781:0:35275:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4762 00000100:00100000:3.0:1783520936.374792:0:35275:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94350626dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0001-mdtlov_UUID+5:3627:x1870152993304960:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1783520936.374831:0:35275:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94350626dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_008:lustre-MDT0001-mdtlov_UUID+5:3627:x1870152993304960:12345-0@lo:400:kworker.0 Request processed in 43us (128us total) trans 0 rc 0/0 00000100:00100000:3.0:1783520936.374838:0:35275:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4762 00000100:00100000:3.0:1783520936.374868:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435118b6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3627:1870152993304960:0@lo:400:kworker.0 00000100:00100000:3.0:1783520936.374876:0:3627:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff943515474700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993305472:0@lo:400:kworker.0 00000100:00100000:3.0:1783520936.374902:0:3627:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1783520936.374939:0:25307:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993305472 00000100:00100000:0.0:1783520936.374943:0:25307:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993305472 00000100:00100000:3.0:1783520936.374947:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3627:1870152993304448:0@lo:400:kworker.0 00000100:00100000:0.0:1783520936.374983:0:25307:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 22663 00000100:00100000:0.0:1783520936.374992:0:25307:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff94351e006d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_004:lustre-MDT0000-mdtlov_UUID+5:3627:x1870152993305472:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1783520936.375039:0:25307:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff94351e006d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_004:lustre-MDT0000-mdtlov_UUID+5:3627:x1870152993305472:12345-0@lo:400:kworker.0 Request processed in 49us (137us total) trans 0 rc 0/0 00000100:00100000:0.0:1783520936.375047:0:25307:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 22663 00000100:00100000:3.0:1783520936.375219:0:3627:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff943515474700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3627:1870152993305472:0@lo:400:kworker.0 00000001:02000400:3.0:1783520936.614449:0:179339:0:(debug.c:654:libcfs_debug_mark_buffer()) DEBUG MARKER: sanityn test_71a: @@@@@@ FAIL: 67 cancel RPC occured. 00000100:00100000:2.0:1783520936.885613:0:3625:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1783520936.885638:0:3625:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9435121a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993305856:0@lo:13:osp-pre-0-0.0 00000100:00100000:2.0:1783520936.885709:0:3625:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1783520936.885759:0:12171:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1870152993305856 00000100:00100000:1.0:1783520936.885770:0:12171:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1870152993305856 00000100:00100000:1.0:1783520936.885822:0:12171:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3194 00000100:00100000:1.0:1783520936.885833:0:12171:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff943514ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0000-mdtlov_UUID+6:3625:x1870152993305856:12345-0@lo:13:osp-pre-0-0.0 00002000:00000024:1.0:1783520936.885856:0:12171:0:(ofd_obd.c:735:ofd_statfs()) blocks cached 0 granted 676312768 pending 0 free 3916488704 avail 3684966400 00000100:00100000:1.0:1783520936.885900:0:12171:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff943514ff1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0000-mdtlov_UUID+6:3625:x1870152993305856:12345-0@lo:13:osp-pre-0-0.0 Request processed in 71us (192us total) trans 0 rc 0/0 00000100:00100000:1.0:1783520936.885906:0:12171:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3194 00000100:00100000:2.0:1783520936.885954:0:3625:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9435121a4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3625:1870152993305856:0@lo:13:osp-pre-0-0.0 00010000:00010000:2.0:1783520936.949513:0:7895:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) Debug log: 58810 lines, 58810 kept, 0 dropped, 0 bad.